{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4fea2309",
   "metadata": {},
   "source": [
    "## 导入所需的包"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "9185f791",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:57:02.358196Z",
     "start_time": "2021-09-02T07:57:02.349954Z"
    }
   },
   "outputs": [],
   "source": [
    "from autox import AutoX\n",
    "from autox.file_io import read_data_from_path\n",
    "from autox.process_data import Feature_type_recognition\n",
    "from autox.util import log, reduce_mem_usage\n",
    "from autox.feature_engineer import FeatureStat\n",
    "from autox.feature_engineer import FeatureCount\n",
    "from autox.models.regressor import CrossLgbRegression\n",
    "from autox.ensemble.stacking import StackingRegressor\n",
    "from autox.models.classifier import CrossLgbBiClassifier, CrossXgbBiClassifier, CrossTabnetBiClassifier\n",
    "import json\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.linear_model import Ridge\n",
    "from lightgbm import LGBMRegressor\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91ebd124",
   "metadata": {},
   "source": [
    "## 初始化AutoX类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d2805fcb",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:40:15.275768Z",
     "start_time": "2021-09-02T07:40:15.271625Z"
    }
   },
   "outputs": [],
   "source": [
    "relations = [\n",
    "    {\n",
    "            \"related_to_main_table\": \"true\", # 是否为和主表的关系\n",
    "            \"left_entity\": \"train_transaction.csv\",  # 左表名字\n",
    "            \"left_on\": [\"TransactionID\"],  # 左表拼表键\n",
    "            \"right_entity\": \"train_identity.csv\",  # 右表名字\n",
    "            \"right_on\": [\"TransactionID\"], # 右表拼表键\n",
    "            \"type\": \"1-1\" # 左表与右表的连接关系\n",
    "        },\n",
    "    {\n",
    "            \"related_to_main_table\": \"true\", # 是否为和主表的关系\n",
    "            \"left_entity\": \"test_transaction.csv\",  # 左表名字\n",
    "            \"left_on\": [\"TransactionID\"],  # 左表拼表键\n",
    "            \"right_entity\": \"test_identity.csv\",  # 右表名字\n",
    "            \"right_on\": [\"TransactionID\"], # 右表拼表键\n",
    "            \"type\": \"1-1\" # 左表与右表的连接关系\n",
    "        }\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3af307da",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.159661Z",
     "start_time": "2021-09-02T07:40:15.277963Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "   INFO ->  [+] read test_identity.csv\n",
      "   INFO ->  Memory usage of dataframe is 44.39 MB\n",
      "   INFO ->  Memory usage after optimization is: 9.84 MB\n",
      "   INFO ->  Decreased by 77.8%\n",
      "   INFO ->  table = test_identity.csv, shape = (141907, 41)\n",
      "   INFO ->  [+] read sample_submission.csv\n",
      "   INFO ->  Memory usage of dataframe is 7.73 MB\n",
      "   INFO ->  Memory usage after optimization is: 2.90 MB\n",
      "   INFO ->  Decreased by 62.5%\n",
      "   INFO ->  table = sample_submission.csv, shape = (506691, 2)\n",
      "   INFO ->  [+] read train_identity.csv\n",
      "   INFO ->  Memory usage of dataframe is 45.12 MB\n",
      "   INFO ->  Memory usage after optimization is: 10.00 MB\n",
      "   INFO ->  Decreased by 77.8%\n",
      "   INFO ->  table = train_identity.csv, shape = (144233, 41)\n",
      "   INFO ->  [+] read test_transaction.csv\n",
      "   INFO ->  Memory usage of dataframe is 1519.24 MB\n",
      "   INFO ->  Memory usage after optimization is: 425.24 MB\n",
      "   INFO ->  Decreased by 72.0%\n",
      "   INFO ->  table = test_transaction.csv, shape = (506691, 393)\n",
      "   INFO ->  [+] read train_transaction.csv\n",
      "   INFO ->  Memory usage of dataframe is 1775.15 MB\n",
      "   INFO ->  Memory usage after optimization is: 487.16 MB\n",
      "   INFO ->  Decreased by 72.6%\n",
      "   INFO ->  table = train_transaction.csv, shape = (590540, 394)\n"
     ]
    }
   ],
   "source": [
    "# 选择数据集\n",
    "data_name = 'kaggle_ieee'\n",
    "path = f'./data/{data_name}'\n",
    "\n",
    "autox = AutoX(target = 'isFraud', train_name = 'train_transaction.csv', test_name = 'test_transaction.csv', \n",
    "               id = ['TransactionID'], path = path, relations = relations)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "48592494",
   "metadata": {},
   "source": [
    "## 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "25fb20b5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.264326Z",
     "start_time": "2021-09-02T07:47:40.261484Z"
    }
   },
   "outputs": [],
   "source": [
    "df = autox.dfs_['train_test']\n",
    "feature_type = autox.info_['feature_type']['train_test']\n",
    "id_ = autox.info_['id']\n",
    "target = autox.info_['target']"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bd093242",
   "metadata": {},
   "source": [
    "### target_encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "a66683c9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.267741Z",
     "start_time": "2021-09-02T07:47:40.265715Z"
    }
   },
   "outputs": [],
   "source": [
    "# from autox.feature_engineer import FeatureTargetEncoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "ea23e352",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.271182Z",
     "start_time": "2021-09-02T07:47:40.269254Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# featureTE = FeatureTargetEncoding()\n",
    "# featureTE.fit(df, 'orders_3h_15h', df_feature_type = feature_type, silence_cols = id_, select_all=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b4b69e3f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.274537Z",
     "start_time": "2021-09-02T07:47:40.272551Z"
    }
   },
   "outputs": [],
   "source": [
    "# log(featureTE.get_ops())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e4b6c3df",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.277734Z",
     "start_time": "2021-09-02T07:47:40.275855Z"
    }
   },
   "outputs": [],
   "source": [
    "# 手动修改配置\n",
    "# featureCount.set_keys([[],[]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "a36ce99e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.281038Z",
     "start_time": "2021-09-02T07:47:40.279085Z"
    }
   },
   "outputs": [],
   "source": [
    "# FE_te = featureTE.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4696065f",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.284456Z",
     "start_time": "2021-09-02T07:47:40.282349Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# FE_te"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53fbafee",
   "metadata": {},
   "source": [
    "### rank特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "81de864a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:40.288015Z",
     "start_time": "2021-09-02T07:47:40.285877Z"
    }
   },
   "outputs": [],
   "source": [
    "from autox.feature_engineer import FeatureRank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1bcba799",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:43.461430Z",
     "start_time": "2021-09-02T07:47:40.289365Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "featureRank = FeatureRank()\n",
    "featureRank.fit(df, df_feature_type = feature_type, select_all = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d0df02cd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:43.468062Z",
     "start_time": "2021-09-02T07:47:43.463322Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4260"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt = 0\n",
    "for key_ in featureRank.get_ops().keys():\n",
    "    cnt += len(featureRank.get_ops()[key_])\n",
    "cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7a63a611",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:43.471892Z",
     "start_time": "2021-09-02T07:47:43.469651Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# FE_rank = featureRank.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "2c96b27a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:43.475589Z",
     "start_time": "2021-09-02T07:47:43.473325Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# FE_rank.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "78a20f19",
   "metadata": {},
   "source": [
    "### 统计特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "592a336d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:46.652441Z",
     "start_time": "2021-09-02T07:47:43.477232Z"
    }
   },
   "outputs": [],
   "source": [
    "featureStat = FeatureStat()\n",
    "featureStat.fit(df, df_feature_type = feature_type, silence_group_cols = id_ + [target], \n",
    "                silence_agg_cols = id_ + [target], select_all = False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "85c0babd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:46.658274Z",
     "start_time": "2021-09-02T07:47:46.654251Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['ProductCD', 'P_emaildomain', 'R_emaildomain', 'id_30', 'id_31', 'id_33', 'DeviceInfo', 'id-30', 'id-31', 'id-33'])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "featureStat.get_ops().keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "eac17a65",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:46.664208Z",
     "start_time": "2021-09-02T07:47:46.659958Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4690"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt = 0\n",
    "for key_ in featureStat.get_ops().keys():\n",
    "    cnt += len(featureStat.get_ops()[key_])\n",
    "cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "e132b164",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:46.670238Z",
     "start_time": "2021-09-02T07:47:46.668019Z"
    }
   },
   "outputs": [],
   "source": [
    "# 手动修改配置\n",
    "# featureGroupby.set_keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "18fe8cc5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:46.674468Z",
     "start_time": "2021-09-02T07:47:46.672131Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# FE_stat = featureStat.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "08e4836d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:46.678332Z",
     "start_time": "2021-09-02T07:47:46.676043Z"
    },
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# FE_stat.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee776428",
   "metadata": {},
   "source": [
    "### count特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "b7eea62e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.802360Z",
     "start_time": "2021-09-02T07:47:46.679779Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "featureCount = FeatureCount()\n",
    "featureCount.fit(df, degree=2, df_feature_type = feature_type, silence_cols = id_ + [target], select_all=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "2d9f482e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.809405Z",
     "start_time": "2021-09-02T07:47:50.804174Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "   INFO ->  [['ProductCD', 'card4'], ['ProductCD', 'card6'], ['ProductCD', 'P_emaildomain'], ['ProductCD', 'R_emaildomain'], ['ProductCD', 'M1'], ['ProductCD', 'M2'], ['ProductCD', 'M3'], ['ProductCD', 'M4'], ['ProductCD', 'M5'], ['ProductCD', 'M6'], ['ProductCD', 'M7'], ['ProductCD', 'M8'], ['ProductCD', 'M9'], ['ProductCD', 'id_12'], ['ProductCD', 'id_15'], ['ProductCD', 'id_16'], ['ProductCD', 'id_23'], ['ProductCD', 'id_27'], ['ProductCD', 'id_28'], ['ProductCD', 'id_29'], ['ProductCD', 'id_30'], ['ProductCD', 'id_31'], ['ProductCD', 'id_33'], ['ProductCD', 'id_34'], ['ProductCD', 'id_35'], ['ProductCD', 'id_36'], ['ProductCD', 'id_37'], ['ProductCD', 'id_38'], ['ProductCD', 'DeviceType'], ['ProductCD', 'DeviceInfo'], ['ProductCD', 'id-12'], ['ProductCD', 'id-15'], ['ProductCD', 'id-16'], ['ProductCD', 'id-23'], ['ProductCD', 'id-27'], ['ProductCD', 'id-28'], ['ProductCD', 'id-29'], ['ProductCD', 'id-30'], ['ProductCD', 'id-31'], ['ProductCD', 'id-33'], ['ProductCD', 'id-34'], ['ProductCD', 'id-35'], ['ProductCD', 'id-36'], ['ProductCD', 'id-37'], ['ProductCD', 'id-38'], ['card4', 'ProductCD'], ['card4', 'card6'], ['card4', 'P_emaildomain'], ['card4', 'R_emaildomain'], ['card4', 'M1'], ['card4', 'M2'], ['card4', 'M3'], ['card4', 'M4'], ['card4', 'M5'], ['card4', 'M6'], ['card4', 'M7'], ['card4', 'M8'], ['card4', 'M9'], ['card4', 'id_12'], ['card4', 'id_15'], ['card4', 'id_16'], ['card4', 'id_23'], ['card4', 'id_27'], ['card4', 'id_28'], ['card4', 'id_29'], ['card4', 'id_30'], ['card4', 'id_31'], ['card4', 'id_33'], ['card4', 'id_34'], ['card4', 'id_35'], ['card4', 'id_36'], ['card4', 'id_37'], ['card4', 'id_38'], ['card4', 'DeviceType'], ['card4', 'DeviceInfo'], ['card4', 'id-12'], ['card4', 'id-15'], ['card4', 'id-16'], ['card4', 'id-23'], ['card4', 'id-27'], ['card4', 'id-28'], ['card4', 'id-29'], ['card4', 'id-30'], ['card4', 'id-31'], ['card4', 'id-33'], ['card4', 'id-34'], ['card4', 'id-35'], ['card4', 'id-36'], ['card4', 'id-37'], ['card4', 'id-38'], ['card6', 'ProductCD'], ['card6', 'card4'], ['card6', 'P_emaildomain'], ['card6', 'R_emaildomain'], ['card6', 'M1'], ['card6', 'M2'], ['card6', 'M3'], ['card6', 'M4'], ['card6', 'M5'], ['card6', 'M6'], ['card6', 'M7'], ['card6', 'M8'], ['card6', 'M9'], ['card6', 'id_12'], ['card6', 'id_15'], ['card6', 'id_16'], ['card6', 'id_23'], ['card6', 'id_27'], ['card6', 'id_28'], ['card6', 'id_29'], ['card6', 'id_30'], ['card6', 'id_31'], ['card6', 'id_33'], ['card6', 'id_34'], ['card6', 'id_35'], ['card6', 'id_36'], ['card6', 'id_37'], ['card6', 'id_38'], ['card6', 'DeviceType'], ['card6', 'DeviceInfo'], ['card6', 'id-12'], ['card6', 'id-15'], ['card6', 'id-16'], ['card6', 'id-23'], ['card6', 'id-27'], ['card6', 'id-28'], ['card6', 'id-29'], ['card6', 'id-30'], ['card6', 'id-31'], ['card6', 'id-33'], ['card6', 'id-34'], ['card6', 'id-35'], ['card6', 'id-36'], ['card6', 'id-37'], ['card6', 'id-38'], ['P_emaildomain', 'ProductCD'], ['P_emaildomain', 'card4'], ['P_emaildomain', 'card6'], ['P_emaildomain', 'R_emaildomain'], ['P_emaildomain', 'M1'], ['P_emaildomain', 'M2'], ['P_emaildomain', 'M3'], ['P_emaildomain', 'M4'], ['P_emaildomain', 'M5'], ['P_emaildomain', 'M6'], ['P_emaildomain', 'M7'], ['P_emaildomain', 'M8'], ['P_emaildomain', 'M9'], ['P_emaildomain', 'id_12'], ['P_emaildomain', 'id_15'], ['P_emaildomain', 'id_16'], ['P_emaildomain', 'id_23'], ['P_emaildomain', 'id_27'], ['P_emaildomain', 'id_28'], ['P_emaildomain', 'id_29'], ['P_emaildomain', 'id_30'], ['P_emaildomain', 'id_31'], ['P_emaildomain', 'id_33'], ['P_emaildomain', 'id_34'], ['P_emaildomain', 'id_35'], ['P_emaildomain', 'id_36'], ['P_emaildomain', 'id_37'], ['P_emaildomain', 'id_38'], ['P_emaildomain', 'DeviceType'], ['P_emaildomain', 'DeviceInfo'], ['P_emaildomain', 'id-12'], ['P_emaildomain', 'id-15'], ['P_emaildomain', 'id-16'], ['P_emaildomain', 'id-23'], ['P_emaildomain', 'id-27'], ['P_emaildomain', 'id-28'], ['P_emaildomain', 'id-29'], ['P_emaildomain', 'id-30'], ['P_emaildomain', 'id-31'], ['P_emaildomain', 'id-33'], ['P_emaildomain', 'id-34'], ['P_emaildomain', 'id-35'], ['P_emaildomain', 'id-36'], ['P_emaildomain', 'id-37'], ['P_emaildomain', 'id-38'], ['R_emaildomain', 'ProductCD'], ['R_emaildomain', 'card4'], ['R_emaildomain', 'card6'], ['R_emaildomain', 'P_emaildomain'], ['R_emaildomain', 'M1'], ['R_emaildomain', 'M2'], ['R_emaildomain', 'M3'], ['R_emaildomain', 'M4'], ['R_emaildomain', 'M5'], ['R_emaildomain', 'M6'], ['R_emaildomain', 'M7'], ['R_emaildomain', 'M8'], ['R_emaildomain', 'M9'], ['R_emaildomain', 'id_12'], ['R_emaildomain', 'id_15'], ['R_emaildomain', 'id_16'], ['R_emaildomain', 'id_23'], ['R_emaildomain', 'id_27'], ['R_emaildomain', 'id_28'], ['R_emaildomain', 'id_29'], ['R_emaildomain', 'id_30'], ['R_emaildomain', 'id_31'], ['R_emaildomain', 'id_33'], ['R_emaildomain', 'id_34'], ['R_emaildomain', 'id_35'], ['R_emaildomain', 'id_36'], ['R_emaildomain', 'id_37'], ['R_emaildomain', 'id_38'], ['R_emaildomain', 'DeviceType'], ['R_emaildomain', 'DeviceInfo'], ['R_emaildomain', 'id-12'], ['R_emaildomain', 'id-15'], ['R_emaildomain', 'id-16'], ['R_emaildomain', 'id-23'], ['R_emaildomain', 'id-27'], ['R_emaildomain', 'id-28'], ['R_emaildomain', 'id-29'], ['R_emaildomain', 'id-30'], ['R_emaildomain', 'id-31'], ['R_emaildomain', 'id-33'], ['R_emaildomain', 'id-34'], ['R_emaildomain', 'id-35'], ['R_emaildomain', 'id-36'], ['R_emaildomain', 'id-37'], ['R_emaildomain', 'id-38'], ['M1', 'ProductCD'], ['M1', 'card4'], ['M1', 'card6'], ['M1', 'P_emaildomain'], ['M1', 'R_emaildomain'], ['M1', 'M2'], ['M1', 'M3'], ['M1', 'M4'], ['M1', 'M5'], ['M1', 'M6'], ['M1', 'M7'], ['M1', 'M8'], ['M1', 'M9'], ['M1', 'id_12'], ['M1', 'id_15'], ['M1', 'id_16'], ['M1', 'id_23'], ['M1', 'id_27'], ['M1', 'id_28'], ['M1', 'id_29'], ['M1', 'id_30'], ['M1', 'id_31'], ['M1', 'id_33'], ['M1', 'id_34'], ['M1', 'id_35'], ['M1', 'id_36'], ['M1', 'id_37'], ['M1', 'id_38'], ['M1', 'DeviceType'], ['M1', 'DeviceInfo'], ['M1', 'id-12'], ['M1', 'id-15'], ['M1', 'id-16'], ['M1', 'id-23'], ['M1', 'id-27'], ['M1', 'id-28'], ['M1', 'id-29'], ['M1', 'id-30'], ['M1', 'id-31'], ['M1', 'id-33'], ['M1', 'id-34'], ['M1', 'id-35'], ['M1', 'id-36'], ['M1', 'id-37'], ['M1', 'id-38'], ['M2', 'ProductCD'], ['M2', 'card4'], ['M2', 'card6'], ['M2', 'P_emaildomain'], ['M2', 'R_emaildomain'], ['M2', 'M1'], ['M2', 'M3'], ['M2', 'M4'], ['M2', 'M5'], ['M2', 'M6'], ['M2', 'M7'], ['M2', 'M8'], ['M2', 'M9'], ['M2', 'id_12'], ['M2', 'id_15'], ['M2', 'id_16'], ['M2', 'id_23'], ['M2', 'id_27'], ['M2', 'id_28'], ['M2', 'id_29'], ['M2', 'id_30'], ['M2', 'id_31'], ['M2', 'id_33'], ['M2', 'id_34'], ['M2', 'id_35'], ['M2', 'id_36'], ['M2', 'id_37'], ['M2', 'id_38'], ['M2', 'DeviceType'], ['M2', 'DeviceInfo'], ['M2', 'id-12'], ['M2', 'id-15'], ['M2', 'id-16'], ['M2', 'id-23'], ['M2', 'id-27'], ['M2', 'id-28'], ['M2', 'id-29'], ['M2', 'id-30'], ['M2', 'id-31'], ['M2', 'id-33'], ['M2', 'id-34'], ['M2', 'id-35'], ['M2', 'id-36'], ['M2', 'id-37'], ['M2', 'id-38'], ['M3', 'ProductCD'], ['M3', 'card4'], ['M3', 'card6'], ['M3', 'P_emaildomain'], ['M3', 'R_emaildomain'], ['M3', 'M1'], ['M3', 'M2'], ['M3', 'M4'], ['M3', 'M5'], ['M3', 'M6'], ['M3', 'M7'], ['M3', 'M8'], ['M3', 'M9'], ['M3', 'id_12'], ['M3', 'id_15'], ['M3', 'id_16'], ['M3', 'id_23'], ['M3', 'id_27'], ['M3', 'id_28'], ['M3', 'id_29'], ['M3', 'id_30'], ['M3', 'id_31'], ['M3', 'id_33'], ['M3', 'id_34'], ['M3', 'id_35'], ['M3', 'id_36'], ['M3', 'id_37'], ['M3', 'id_38'], ['M3', 'DeviceType'], ['M3', 'DeviceInfo'], ['M3', 'id-12'], ['M3', 'id-15'], ['M3', 'id-16'], ['M3', 'id-23'], ['M3', 'id-27'], ['M3', 'id-28'], ['M3', 'id-29'], ['M3', 'id-30'], ['M3', 'id-31'], ['M3', 'id-33'], ['M3', 'id-34'], ['M3', 'id-35'], ['M3', 'id-36'], ['M3', 'id-37'], ['M3', 'id-38'], ['M4', 'ProductCD'], ['M4', 'card4'], ['M4', 'card6'], ['M4', 'P_emaildomain'], ['M4', 'R_emaildomain'], ['M4', 'M1'], ['M4', 'M2'], ['M4', 'M3'], ['M4', 'M5'], ['M4', 'M6'], ['M4', 'M7'], ['M4', 'M8'], ['M4', 'M9'], ['M4', 'id_12'], ['M4', 'id_15'], ['M4', 'id_16'], ['M4', 'id_23'], ['M4', 'id_27'], ['M4', 'id_28'], ['M4', 'id_29'], ['M4', 'id_30'], ['M4', 'id_31'], ['M4', 'id_33'], ['M4', 'id_34'], ['M4', 'id_35'], ['M4', 'id_36'], ['M4', 'id_37'], ['M4', 'id_38'], ['M4', 'DeviceType'], ['M4', 'DeviceInfo'], ['M4', 'id-12'], ['M4', 'id-15'], ['M4', 'id-16'], ['M4', 'id-23'], ['M4', 'id-27'], ['M4', 'id-28'], ['M4', 'id-29'], ['M4', 'id-30'], ['M4', 'id-31'], ['M4', 'id-33'], ['M4', 'id-34'], ['M4', 'id-35'], ['M4', 'id-36'], ['M4', 'id-37'], ['M4', 'id-38'], ['M5', 'ProductCD'], ['M5', 'card4'], ['M5', 'card6'], ['M5', 'P_emaildomain'], ['M5', 'R_emaildomain'], ['M5', 'M1'], ['M5', 'M2'], ['M5', 'M3'], ['M5', 'M4'], ['M5', 'M6'], ['M5', 'M7'], ['M5', 'M8'], ['M5', 'M9'], ['M5', 'id_12'], ['M5', 'id_15'], ['M5', 'id_16'], ['M5', 'id_23'], ['M5', 'id_27'], ['M5', 'id_28'], ['M5', 'id_29'], ['M5', 'id_30'], ['M5', 'id_31'], ['M5', 'id_33'], ['M5', 'id_34'], ['M5', 'id_35'], ['M5', 'id_36'], ['M5', 'id_37'], ['M5', 'id_38'], ['M5', 'DeviceType'], ['M5', 'DeviceInfo'], ['M5', 'id-12'], ['M5', 'id-15'], ['M5', 'id-16'], ['M5', 'id-23'], ['M5', 'id-27'], ['M5', 'id-28'], ['M5', 'id-29'], ['M5', 'id-30'], ['M5', 'id-31'], ['M5', 'id-33'], ['M5', 'id-34'], ['M5', 'id-35'], ['M5', 'id-36'], ['M5', 'id-37'], ['M5', 'id-38'], ['M6', 'ProductCD'], ['M6', 'card4'], ['M6', 'card6'], ['M6', 'P_emaildomain'], ['M6', 'R_emaildomain'], ['M6', 'M1'], ['M6', 'M2'], ['M6', 'M3'], ['M6', 'M4'], ['M6', 'M5'], ['M6', 'M7'], ['M6', 'M8'], ['M6', 'M9'], ['M6', 'id_12'], ['M6', 'id_15'], ['M6', 'id_16'], ['M6', 'id_23'], ['M6', 'id_27'], ['M6', 'id_28'], ['M6', 'id_29'], ['M6', 'id_30'], ['M6', 'id_31'], ['M6', 'id_33'], ['M6', 'id_34'], ['M6', 'id_35'], ['M6', 'id_36'], ['M6', 'id_37'], ['M6', 'id_38'], ['M6', 'DeviceType'], ['M6', 'DeviceInfo'], ['M6', 'id-12'], ['M6', 'id-15'], ['M6', 'id-16'], ['M6', 'id-23'], ['M6', 'id-27'], ['M6', 'id-28'], ['M6', 'id-29'], ['M6', 'id-30'], ['M6', 'id-31'], ['M6', 'id-33'], ['M6', 'id-34'], ['M6', 'id-35'], ['M6', 'id-36'], ['M6', 'id-37'], ['M6', 'id-38'], ['M7', 'ProductCD'], ['M7', 'card4'], ['M7', 'card6'], ['M7', 'P_emaildomain'], ['M7', 'R_emaildomain'], ['M7', 'M1'], ['M7', 'M2'], ['M7', 'M3'], ['M7', 'M4'], ['M7', 'M5'], ['M7', 'M6'], ['M7', 'M8'], ['M7', 'M9'], ['M7', 'id_12'], ['M7', 'id_15'], ['M7', 'id_16'], ['M7', 'id_23'], ['M7', 'id_27'], ['M7', 'id_28'], ['M7', 'id_29'], ['M7', 'id_30'], ['M7', 'id_31'], ['M7', 'id_33'], ['M7', 'id_34'], ['M7', 'id_35'], ['M7', 'id_36'], ['M7', 'id_37'], ['M7', 'id_38'], ['M7', 'DeviceType'], ['M7', 'DeviceInfo'], ['M7', 'id-12'], ['M7', 'id-15'], ['M7', 'id-16'], ['M7', 'id-23'], ['M7', 'id-27'], ['M7', 'id-28'], ['M7', 'id-29'], ['M7', 'id-30'], ['M7', 'id-31'], ['M7', 'id-33'], ['M7', 'id-34'], ['M7', 'id-35'], ['M7', 'id-36'], ['M7', 'id-37'], ['M7', 'id-38'], ['M8', 'ProductCD'], ['M8', 'card4'], ['M8', 'card6'], ['M8', 'P_emaildomain'], ['M8', 'R_emaildomain'], ['M8', 'M1'], ['M8', 'M2'], ['M8', 'M3'], ['M8', 'M4'], ['M8', 'M5'], ['M8', 'M6'], ['M8', 'M7'], ['M8', 'M9'], ['M8', 'id_12'], ['M8', 'id_15'], ['M8', 'id_16'], ['M8', 'id_23'], ['M8', 'id_27'], ['M8', 'id_28'], ['M8', 'id_29'], ['M8', 'id_30'], ['M8', 'id_31'], ['M8', 'id_33'], ['M8', 'id_34'], ['M8', 'id_35'], ['M8', 'id_36'], ['M8', 'id_37'], ['M8', 'id_38'], ['M8', 'DeviceType'], ['M8', 'DeviceInfo'], ['M8', 'id-12'], ['M8', 'id-15'], ['M8', 'id-16'], ['M8', 'id-23'], ['M8', 'id-27'], ['M8', 'id-28'], ['M8', 'id-29'], ['M8', 'id-30'], ['M8', 'id-31'], ['M8', 'id-33'], ['M8', 'id-34'], ['M8', 'id-35'], ['M8', 'id-36'], ['M8', 'id-37'], ['M8', 'id-38'], ['M9', 'ProductCD'], ['M9', 'card4'], ['M9', 'card6'], ['M9', 'P_emaildomain'], ['M9', 'R_emaildomain'], ['M9', 'M1'], ['M9', 'M2'], ['M9', 'M3'], ['M9', 'M4'], ['M9', 'M5'], ['M9', 'M6'], ['M9', 'M7'], ['M9', 'M8'], ['M9', 'id_12'], ['M9', 'id_15'], ['M9', 'id_16'], ['M9', 'id_23'], ['M9', 'id_27'], ['M9', 'id_28'], ['M9', 'id_29'], ['M9', 'id_30'], ['M9', 'id_31'], ['M9', 'id_33'], ['M9', 'id_34'], ['M9', 'id_35'], ['M9', 'id_36'], ['M9', 'id_37'], ['M9', 'id_38'], ['M9', 'DeviceType'], ['M9', 'DeviceInfo'], ['M9', 'id-12'], ['M9', 'id-15'], ['M9', 'id-16'], ['M9', 'id-23'], ['M9', 'id-27'], ['M9', 'id-28'], ['M9', 'id-29'], ['M9', 'id-30'], ['M9', 'id-31'], ['M9', 'id-33'], ['M9', 'id-34'], ['M9', 'id-35'], ['M9', 'id-36'], ['M9', 'id-37'], ['M9', 'id-38'], ['id_12', 'ProductCD'], ['id_12', 'card4'], ['id_12', 'card6'], ['id_12', 'P_emaildomain'], ['id_12', 'R_emaildomain'], ['id_12', 'M1'], ['id_12', 'M2'], ['id_12', 'M3'], ['id_12', 'M4'], ['id_12', 'M5'], ['id_12', 'M6'], ['id_12', 'M7'], ['id_12', 'M8'], ['id_12', 'M9'], ['id_12', 'id_15'], ['id_12', 'id_16'], ['id_12', 'id_23'], ['id_12', 'id_27'], ['id_12', 'id_28'], ['id_12', 'id_29'], ['id_12', 'id_30'], ['id_12', 'id_31'], ['id_12', 'id_33'], ['id_12', 'id_34'], ['id_12', 'id_35'], ['id_12', 'id_36'], ['id_12', 'id_37'], ['id_12', 'id_38'], ['id_12', 'DeviceType'], ['id_12', 'DeviceInfo'], ['id_12', 'id-12'], ['id_12', 'id-15'], ['id_12', 'id-16'], ['id_12', 'id-23'], ['id_12', 'id-27'], ['id_12', 'id-28'], ['id_12', 'id-29'], ['id_12', 'id-30'], ['id_12', 'id-31'], ['id_12', 'id-33'], ['id_12', 'id-34'], ['id_12', 'id-35'], ['id_12', 'id-36'], ['id_12', 'id-37'], ['id_12', 'id-38'], ['id_15', 'ProductCD'], ['id_15', 'card4'], ['id_15', 'card6'], ['id_15', 'P_emaildomain'], ['id_15', 'R_emaildomain'], ['id_15', 'M1'], ['id_15', 'M2'], ['id_15', 'M3'], ['id_15', 'M4'], ['id_15', 'M5'], ['id_15', 'M6'], ['id_15', 'M7'], ['id_15', 'M8'], ['id_15', 'M9'], ['id_15', 'id_12'], ['id_15', 'id_16'], ['id_15', 'id_23'], ['id_15', 'id_27'], ['id_15', 'id_28'], ['id_15', 'id_29'], ['id_15', 'id_30'], ['id_15', 'id_31'], ['id_15', 'id_33'], ['id_15', 'id_34'], ['id_15', 'id_35'], ['id_15', 'id_36'], ['id_15', 'id_37'], ['id_15', 'id_38'], ['id_15', 'DeviceType'], ['id_15', 'DeviceInfo'], ['id_15', 'id-12'], ['id_15', 'id-15'], ['id_15', 'id-16'], ['id_15', 'id-23'], ['id_15', 'id-27'], ['id_15', 'id-28'], ['id_15', 'id-29'], ['id_15', 'id-30'], ['id_15', 'id-31'], ['id_15', 'id-33'], ['id_15', 'id-34'], ['id_15', 'id-35'], ['id_15', 'id-36'], ['id_15', 'id-37'], ['id_15', 'id-38'], ['id_16', 'ProductCD'], ['id_16', 'card4'], ['id_16', 'card6'], ['id_16', 'P_emaildomain'], ['id_16', 'R_emaildomain'], ['id_16', 'M1'], ['id_16', 'M2'], ['id_16', 'M3'], ['id_16', 'M4'], ['id_16', 'M5'], ['id_16', 'M6'], ['id_16', 'M7'], ['id_16', 'M8'], ['id_16', 'M9'], ['id_16', 'id_12'], ['id_16', 'id_15'], ['id_16', 'id_23'], ['id_16', 'id_27'], ['id_16', 'id_28'], ['id_16', 'id_29'], ['id_16', 'id_30'], ['id_16', 'id_31'], ['id_16', 'id_33'], ['id_16', 'id_34'], ['id_16', 'id_35'], ['id_16', 'id_36'], ['id_16', 'id_37'], ['id_16', 'id_38'], ['id_16', 'DeviceType'], ['id_16', 'DeviceInfo'], ['id_16', 'id-12'], ['id_16', 'id-15'], ['id_16', 'id-16'], ['id_16', 'id-23'], ['id_16', 'id-27'], ['id_16', 'id-28'], ['id_16', 'id-29'], ['id_16', 'id-30'], ['id_16', 'id-31'], ['id_16', 'id-33'], ['id_16', 'id-34'], ['id_16', 'id-35'], ['id_16', 'id-36'], ['id_16', 'id-37'], ['id_16', 'id-38'], ['id_23', 'ProductCD'], ['id_23', 'card4'], ['id_23', 'card6'], ['id_23', 'P_emaildomain'], ['id_23', 'R_emaildomain'], ['id_23', 'M1'], ['id_23', 'M2'], ['id_23', 'M3'], ['id_23', 'M4'], ['id_23', 'M5'], ['id_23', 'M6'], ['id_23', 'M7'], ['id_23', 'M8'], ['id_23', 'M9'], ['id_23', 'id_12'], ['id_23', 'id_15'], ['id_23', 'id_16'], ['id_23', 'id_27'], ['id_23', 'id_28'], ['id_23', 'id_29'], ['id_23', 'id_30'], ['id_23', 'id_31'], ['id_23', 'id_33'], ['id_23', 'id_34'], ['id_23', 'id_35'], ['id_23', 'id_36'], ['id_23', 'id_37'], ['id_23', 'id_38'], ['id_23', 'DeviceType'], ['id_23', 'DeviceInfo'], ['id_23', 'id-12'], ['id_23', 'id-15'], ['id_23', 'id-16'], ['id_23', 'id-23'], ['id_23', 'id-27'], ['id_23', 'id-28'], ['id_23', 'id-29'], ['id_23', 'id-30'], ['id_23', 'id-31'], ['id_23', 'id-33'], ['id_23', 'id-34'], ['id_23', 'id-35'], ['id_23', 'id-36'], ['id_23', 'id-37'], ['id_23', 'id-38'], ['id_27', 'ProductCD'], ['id_27', 'card4'], ['id_27', 'card6'], ['id_27', 'P_emaildomain'], ['id_27', 'R_emaildomain'], ['id_27', 'M1'], ['id_27', 'M2'], ['id_27', 'M3'], ['id_27', 'M4'], ['id_27', 'M5'], ['id_27', 'M6'], ['id_27', 'M7'], ['id_27', 'M8'], ['id_27', 'M9'], ['id_27', 'id_12'], ['id_27', 'id_15'], ['id_27', 'id_16'], ['id_27', 'id_23'], ['id_27', 'id_28'], ['id_27', 'id_29'], ['id_27', 'id_30'], ['id_27', 'id_31'], ['id_27', 'id_33'], ['id_27', 'id_34'], ['id_27', 'id_35'], ['id_27', 'id_36'], ['id_27', 'id_37'], ['id_27', 'id_38'], ['id_27', 'DeviceType'], ['id_27', 'DeviceInfo'], ['id_27', 'id-12'], ['id_27', 'id-15'], ['id_27', 'id-16'], ['id_27', 'id-23'], ['id_27', 'id-27'], ['id_27', 'id-28'], ['id_27', 'id-29'], ['id_27', 'id-30'], ['id_27', 'id-31'], ['id_27', 'id-33'], ['id_27', 'id-34'], ['id_27', 'id-35'], ['id_27', 'id-36'], ['id_27', 'id-37'], ['id_27', 'id-38'], ['id_28', 'ProductCD'], ['id_28', 'card4'], ['id_28', 'card6'], ['id_28', 'P_emaildomain'], ['id_28', 'R_emaildomain'], ['id_28', 'M1'], ['id_28', 'M2'], ['id_28', 'M3'], ['id_28', 'M4'], ['id_28', 'M5'], ['id_28', 'M6'], ['id_28', 'M7'], ['id_28', 'M8'], ['id_28', 'M9'], ['id_28', 'id_12'], ['id_28', 'id_15'], ['id_28', 'id_16'], ['id_28', 'id_23'], ['id_28', 'id_27'], ['id_28', 'id_29'], ['id_28', 'id_30'], ['id_28', 'id_31'], ['id_28', 'id_33'], ['id_28', 'id_34'], ['id_28', 'id_35'], ['id_28', 'id_36'], ['id_28', 'id_37'], ['id_28', 'id_38'], ['id_28', 'DeviceType'], ['id_28', 'DeviceInfo'], ['id_28', 'id-12'], ['id_28', 'id-15'], ['id_28', 'id-16'], ['id_28', 'id-23'], ['id_28', 'id-27'], ['id_28', 'id-28'], ['id_28', 'id-29'], ['id_28', 'id-30'], ['id_28', 'id-31'], ['id_28', 'id-33'], ['id_28', 'id-34'], ['id_28', 'id-35'], ['id_28', 'id-36'], ['id_28', 'id-37'], ['id_28', 'id-38'], ['id_29', 'ProductCD'], ['id_29', 'card4'], ['id_29', 'card6'], ['id_29', 'P_emaildomain'], ['id_29', 'R_emaildomain'], ['id_29', 'M1'], ['id_29', 'M2'], ['id_29', 'M3'], ['id_29', 'M4'], ['id_29', 'M5'], ['id_29', 'M6'], ['id_29', 'M7'], ['id_29', 'M8'], ['id_29', 'M9'], ['id_29', 'id_12'], ['id_29', 'id_15'], ['id_29', 'id_16'], ['id_29', 'id_23'], ['id_29', 'id_27'], ['id_29', 'id_28'], ['id_29', 'id_30'], ['id_29', 'id_31'], ['id_29', 'id_33'], ['id_29', 'id_34'], ['id_29', 'id_35'], ['id_29', 'id_36'], ['id_29', 'id_37'], ['id_29', 'id_38'], ['id_29', 'DeviceType'], ['id_29', 'DeviceInfo'], ['id_29', 'id-12'], ['id_29', 'id-15'], ['id_29', 'id-16'], ['id_29', 'id-23'], ['id_29', 'id-27'], ['id_29', 'id-28'], ['id_29', 'id-29'], ['id_29', 'id-30'], ['id_29', 'id-31'], ['id_29', 'id-33'], ['id_29', 'id-34'], ['id_29', 'id-35'], ['id_29', 'id-36'], ['id_29', 'id-37'], ['id_29', 'id-38'], ['id_30', 'ProductCD'], ['id_30', 'card4'], ['id_30', 'card6'], ['id_30', 'P_emaildomain'], ['id_30', 'R_emaildomain'], ['id_30', 'M1'], ['id_30', 'M2'], ['id_30', 'M3'], ['id_30', 'M4'], ['id_30', 'M5'], ['id_30', 'M6'], ['id_30', 'M7'], ['id_30', 'M8'], ['id_30', 'M9'], ['id_30', 'id_12'], ['id_30', 'id_15'], ['id_30', 'id_16'], ['id_30', 'id_23'], ['id_30', 'id_27'], ['id_30', 'id_28'], ['id_30', 'id_29'], ['id_30', 'id_31'], ['id_30', 'id_33'], ['id_30', 'id_34'], ['id_30', 'id_35'], ['id_30', 'id_36'], ['id_30', 'id_37'], ['id_30', 'id_38'], ['id_30', 'DeviceType'], ['id_30', 'DeviceInfo'], ['id_30', 'id-12'], ['id_30', 'id-15'], ['id_30', 'id-16'], ['id_30', 'id-23'], ['id_30', 'id-27'], ['id_30', 'id-28'], ['id_30', 'id-29'], ['id_30', 'id-30'], ['id_30', 'id-31'], ['id_30', 'id-33'], ['id_30', 'id-34'], ['id_30', 'id-35'], ['id_30', 'id-36'], ['id_30', 'id-37'], ['id_30', 'id-38'], ['id_31', 'ProductCD'], ['id_31', 'card4'], ['id_31', 'card6'], ['id_31', 'P_emaildomain'], ['id_31', 'R_emaildomain'], ['id_31', 'M1'], ['id_31', 'M2'], ['id_31', 'M3'], ['id_31', 'M4'], ['id_31', 'M5'], ['id_31', 'M6'], ['id_31', 'M7'], ['id_31', 'M8'], ['id_31', 'M9'], ['id_31', 'id_12'], ['id_31', 'id_15'], ['id_31', 'id_16'], ['id_31', 'id_23'], ['id_31', 'id_27'], ['id_31', 'id_28'], ['id_31', 'id_29'], ['id_31', 'id_30'], ['id_31', 'id_33'], ['id_31', 'id_34'], ['id_31', 'id_35'], ['id_31', 'id_36'], ['id_31', 'id_37'], ['id_31', 'id_38'], ['id_31', 'DeviceType'], ['id_31', 'DeviceInfo'], ['id_31', 'id-12'], ['id_31', 'id-15'], ['id_31', 'id-16'], ['id_31', 'id-23'], ['id_31', 'id-27'], ['id_31', 'id-28'], ['id_31', 'id-29'], ['id_31', 'id-30'], ['id_31', 'id-31'], ['id_31', 'id-33'], ['id_31', 'id-34'], ['id_31', 'id-35'], ['id_31', 'id-36'], ['id_31', 'id-37'], ['id_31', 'id-38'], ['id_33', 'ProductCD'], ['id_33', 'card4'], ['id_33', 'card6'], ['id_33', 'P_emaildomain'], ['id_33', 'R_emaildomain'], ['id_33', 'M1'], ['id_33', 'M2'], ['id_33', 'M3'], ['id_33', 'M4'], ['id_33', 'M5'], ['id_33', 'M6'], ['id_33', 'M7'], ['id_33', 'M8'], ['id_33', 'M9'], ['id_33', 'id_12'], ['id_33', 'id_15'], ['id_33', 'id_16'], ['id_33', 'id_23'], ['id_33', 'id_27'], ['id_33', 'id_28'], ['id_33', 'id_29'], ['id_33', 'id_30'], ['id_33', 'id_31'], ['id_33', 'id_34'], ['id_33', 'id_35'], ['id_33', 'id_36'], ['id_33', 'id_37'], ['id_33', 'id_38'], ['id_33', 'DeviceType'], ['id_33', 'DeviceInfo'], ['id_33', 'id-12'], ['id_33', 'id-15'], ['id_33', 'id-16'], ['id_33', 'id-23'], ['id_33', 'id-27'], ['id_33', 'id-28'], ['id_33', 'id-29'], ['id_33', 'id-30'], ['id_33', 'id-31'], ['id_33', 'id-33'], ['id_33', 'id-34'], ['id_33', 'id-35'], ['id_33', 'id-36'], ['id_33', 'id-37'], ['id_33', 'id-38'], ['id_34', 'ProductCD'], ['id_34', 'card4'], ['id_34', 'card6'], ['id_34', 'P_emaildomain'], ['id_34', 'R_emaildomain'], ['id_34', 'M1'], ['id_34', 'M2'], ['id_34', 'M3'], ['id_34', 'M4'], ['id_34', 'M5'], ['id_34', 'M6'], ['id_34', 'M7'], ['id_34', 'M8'], ['id_34', 'M9'], ['id_34', 'id_12'], ['id_34', 'id_15'], ['id_34', 'id_16'], ['id_34', 'id_23'], ['id_34', 'id_27'], ['id_34', 'id_28'], ['id_34', 'id_29'], ['id_34', 'id_30'], ['id_34', 'id_31'], ['id_34', 'id_33'], ['id_34', 'id_35'], ['id_34', 'id_36'], ['id_34', 'id_37'], ['id_34', 'id_38'], ['id_34', 'DeviceType'], ['id_34', 'DeviceInfo'], ['id_34', 'id-12'], ['id_34', 'id-15'], ['id_34', 'id-16'], ['id_34', 'id-23'], ['id_34', 'id-27'], ['id_34', 'id-28'], ['id_34', 'id-29'], ['id_34', 'id-30'], ['id_34', 'id-31'], ['id_34', 'id-33'], ['id_34', 'id-34'], ['id_34', 'id-35'], ['id_34', 'id-36'], ['id_34', 'id-37'], ['id_34', 'id-38'], ['id_35', 'ProductCD'], ['id_35', 'card4'], ['id_35', 'card6'], ['id_35', 'P_emaildomain'], ['id_35', 'R_emaildomain'], ['id_35', 'M1'], ['id_35', 'M2'], ['id_35', 'M3'], ['id_35', 'M4'], ['id_35', 'M5'], ['id_35', 'M6'], ['id_35', 'M7'], ['id_35', 'M8'], ['id_35', 'M9'], ['id_35', 'id_12'], ['id_35', 'id_15'], ['id_35', 'id_16'], ['id_35', 'id_23'], ['id_35', 'id_27'], ['id_35', 'id_28'], ['id_35', 'id_29'], ['id_35', 'id_30'], ['id_35', 'id_31'], ['id_35', 'id_33'], ['id_35', 'id_34'], ['id_35', 'id_36'], ['id_35', 'id_37'], ['id_35', 'id_38'], ['id_35', 'DeviceType'], ['id_35', 'DeviceInfo'], ['id_35', 'id-12'], ['id_35', 'id-15'], ['id_35', 'id-16'], ['id_35', 'id-23'], ['id_35', 'id-27'], ['id_35', 'id-28'], ['id_35', 'id-29'], ['id_35', 'id-30'], ['id_35', 'id-31'], ['id_35', 'id-33'], ['id_35', 'id-34'], ['id_35', 'id-35'], ['id_35', 'id-36'], ['id_35', 'id-37'], ['id_35', 'id-38'], ['id_36', 'ProductCD'], ['id_36', 'card4'], ['id_36', 'card6'], ['id_36', 'P_emaildomain'], ['id_36', 'R_emaildomain'], ['id_36', 'M1'], ['id_36', 'M2'], ['id_36', 'M3'], ['id_36', 'M4'], ['id_36', 'M5'], ['id_36', 'M6'], ['id_36', 'M7'], ['id_36', 'M8'], ['id_36', 'M9'], ['id_36', 'id_12'], ['id_36', 'id_15'], ['id_36', 'id_16'], ['id_36', 'id_23'], ['id_36', 'id_27'], ['id_36', 'id_28'], ['id_36', 'id_29'], ['id_36', 'id_30'], ['id_36', 'id_31'], ['id_36', 'id_33'], ['id_36', 'id_34'], ['id_36', 'id_35'], ['id_36', 'id_37'], ['id_36', 'id_38'], ['id_36', 'DeviceType'], ['id_36', 'DeviceInfo'], ['id_36', 'id-12'], ['id_36', 'id-15'], ['id_36', 'id-16'], ['id_36', 'id-23'], ['id_36', 'id-27'], ['id_36', 'id-28'], ['id_36', 'id-29'], ['id_36', 'id-30'], ['id_36', 'id-31'], ['id_36', 'id-33'], ['id_36', 'id-34'], ['id_36', 'id-35'], ['id_36', 'id-36'], ['id_36', 'id-37'], ['id_36', 'id-38'], ['id_37', 'ProductCD'], ['id_37', 'card4'], ['id_37', 'card6'], ['id_37', 'P_emaildomain'], ['id_37', 'R_emaildomain'], ['id_37', 'M1'], ['id_37', 'M2'], ['id_37', 'M3'], ['id_37', 'M4'], ['id_37', 'M5'], ['id_37', 'M6'], ['id_37', 'M7'], ['id_37', 'M8'], ['id_37', 'M9'], ['id_37', 'id_12'], ['id_37', 'id_15'], ['id_37', 'id_16'], ['id_37', 'id_23'], ['id_37', 'id_27'], ['id_37', 'id_28'], ['id_37', 'id_29'], ['id_37', 'id_30'], ['id_37', 'id_31'], ['id_37', 'id_33'], ['id_37', 'id_34'], ['id_37', 'id_35'], ['id_37', 'id_36'], ['id_37', 'id_38'], ['id_37', 'DeviceType'], ['id_37', 'DeviceInfo'], ['id_37', 'id-12'], ['id_37', 'id-15'], ['id_37', 'id-16'], ['id_37', 'id-23'], ['id_37', 'id-27'], ['id_37', 'id-28'], ['id_37', 'id-29'], ['id_37', 'id-30'], ['id_37', 'id-31'], ['id_37', 'id-33'], ['id_37', 'id-34'], ['id_37', 'id-35'], ['id_37', 'id-36'], ['id_37', 'id-37'], ['id_37', 'id-38'], ['id_38', 'ProductCD'], ['id_38', 'card4'], ['id_38', 'card6'], ['id_38', 'P_emaildomain'], ['id_38', 'R_emaildomain'], ['id_38', 'M1'], ['id_38', 'M2'], ['id_38', 'M3'], ['id_38', 'M4'], ['id_38', 'M5'], ['id_38', 'M6'], ['id_38', 'M7'], ['id_38', 'M8'], ['id_38', 'M9'], ['id_38', 'id_12'], ['id_38', 'id_15'], ['id_38', 'id_16'], ['id_38', 'id_23'], ['id_38', 'id_27'], ['id_38', 'id_28'], ['id_38', 'id_29'], ['id_38', 'id_30'], ['id_38', 'id_31'], ['id_38', 'id_33'], ['id_38', 'id_34'], ['id_38', 'id_35'], ['id_38', 'id_36'], ['id_38', 'id_37'], ['id_38', 'DeviceType'], ['id_38', 'DeviceInfo'], ['id_38', 'id-12'], ['id_38', 'id-15'], ['id_38', 'id-16'], ['id_38', 'id-23'], ['id_38', 'id-27'], ['id_38', 'id-28'], ['id_38', 'id-29'], ['id_38', 'id-30'], ['id_38', 'id-31'], ['id_38', 'id-33'], ['id_38', 'id-34'], ['id_38', 'id-35'], ['id_38', 'id-36'], ['id_38', 'id-37'], ['id_38', 'id-38'], ['DeviceType', 'ProductCD'], ['DeviceType', 'card4'], ['DeviceType', 'card6'], ['DeviceType', 'P_emaildomain'], ['DeviceType', 'R_emaildomain'], ['DeviceType', 'M1'], ['DeviceType', 'M2'], ['DeviceType', 'M3'], ['DeviceType', 'M4'], ['DeviceType', 'M5'], ['DeviceType', 'M6'], ['DeviceType', 'M7'], ['DeviceType', 'M8'], ['DeviceType', 'M9'], ['DeviceType', 'id_12'], ['DeviceType', 'id_15'], ['DeviceType', 'id_16'], ['DeviceType', 'id_23'], ['DeviceType', 'id_27'], ['DeviceType', 'id_28'], ['DeviceType', 'id_29'], ['DeviceType', 'id_30'], ['DeviceType', 'id_31'], ['DeviceType', 'id_33'], ['DeviceType', 'id_34'], ['DeviceType', 'id_35'], ['DeviceType', 'id_36'], ['DeviceType', 'id_37'], ['DeviceType', 'id_38'], ['DeviceType', 'DeviceInfo'], ['DeviceType', 'id-12'], ['DeviceType', 'id-15'], ['DeviceType', 'id-16'], ['DeviceType', 'id-23'], ['DeviceType', 'id-27'], ['DeviceType', 'id-28'], ['DeviceType', 'id-29'], ['DeviceType', 'id-30'], ['DeviceType', 'id-31'], ['DeviceType', 'id-33'], ['DeviceType', 'id-34'], ['DeviceType', 'id-35'], ['DeviceType', 'id-36'], ['DeviceType', 'id-37'], ['DeviceType', 'id-38'], ['DeviceInfo', 'ProductCD'], ['DeviceInfo', 'card4'], ['DeviceInfo', 'card6'], ['DeviceInfo', 'P_emaildomain'], ['DeviceInfo', 'R_emaildomain'], ['DeviceInfo', 'M1'], ['DeviceInfo', 'M2'], ['DeviceInfo', 'M3'], ['DeviceInfo', 'M4'], ['DeviceInfo', 'M5'], ['DeviceInfo', 'M6'], ['DeviceInfo', 'M7'], ['DeviceInfo', 'M8'], ['DeviceInfo', 'M9'], ['DeviceInfo', 'id_12'], ['DeviceInfo', 'id_15'], ['DeviceInfo', 'id_16'], ['DeviceInfo', 'id_23'], ['DeviceInfo', 'id_27'], ['DeviceInfo', 'id_28'], ['DeviceInfo', 'id_29'], ['DeviceInfo', 'id_30'], ['DeviceInfo', 'id_31'], ['DeviceInfo', 'id_33'], ['DeviceInfo', 'id_34'], ['DeviceInfo', 'id_35'], ['DeviceInfo', 'id_36'], ['DeviceInfo', 'id_37'], ['DeviceInfo', 'id_38'], ['DeviceInfo', 'DeviceType'], ['DeviceInfo', 'id-12'], ['DeviceInfo', 'id-15'], ['DeviceInfo', 'id-16'], ['DeviceInfo', 'id-23'], ['DeviceInfo', 'id-27'], ['DeviceInfo', 'id-28'], ['DeviceInfo', 'id-29'], ['DeviceInfo', 'id-30'], ['DeviceInfo', 'id-31'], ['DeviceInfo', 'id-33'], ['DeviceInfo', 'id-34'], ['DeviceInfo', 'id-35'], ['DeviceInfo', 'id-36'], ['DeviceInfo', 'id-37'], ['DeviceInfo', 'id-38'], ['id-12', 'ProductCD'], ['id-12', 'card4'], ['id-12', 'card6'], ['id-12', 'P_emaildomain'], ['id-12', 'R_emaildomain'], ['id-12', 'M1'], ['id-12', 'M2'], ['id-12', 'M3'], ['id-12', 'M4'], ['id-12', 'M5'], ['id-12', 'M6'], ['id-12', 'M7'], ['id-12', 'M8'], ['id-12', 'M9'], ['id-12', 'id_12'], ['id-12', 'id_15'], ['id-12', 'id_16'], ['id-12', 'id_23'], ['id-12', 'id_27'], ['id-12', 'id_28'], ['id-12', 'id_29'], ['id-12', 'id_30'], ['id-12', 'id_31'], ['id-12', 'id_33'], ['id-12', 'id_34'], ['id-12', 'id_35'], ['id-12', 'id_36'], ['id-12', 'id_37'], ['id-12', 'id_38'], ['id-12', 'DeviceType'], ['id-12', 'DeviceInfo'], ['id-12', 'id-15'], ['id-12', 'id-16'], ['id-12', 'id-23'], ['id-12', 'id-27'], ['id-12', 'id-28'], ['id-12', 'id-29'], ['id-12', 'id-30'], ['id-12', 'id-31'], ['id-12', 'id-33'], ['id-12', 'id-34'], ['id-12', 'id-35'], ['id-12', 'id-36'], ['id-12', 'id-37'], ['id-12', 'id-38'], ['id-15', 'ProductCD'], ['id-15', 'card4'], ['id-15', 'card6'], ['id-15', 'P_emaildomain'], ['id-15', 'R_emaildomain'], ['id-15', 'M1'], ['id-15', 'M2'], ['id-15', 'M3'], ['id-15', 'M4'], ['id-15', 'M5'], ['id-15', 'M6'], ['id-15', 'M7'], ['id-15', 'M8'], ['id-15', 'M9'], ['id-15', 'id_12'], ['id-15', 'id_15'], ['id-15', 'id_16'], ['id-15', 'id_23'], ['id-15', 'id_27'], ['id-15', 'id_28'], ['id-15', 'id_29'], ['id-15', 'id_30'], ['id-15', 'id_31'], ['id-15', 'id_33'], ['id-15', 'id_34'], ['id-15', 'id_35'], ['id-15', 'id_36'], ['id-15', 'id_37'], ['id-15', 'id_38'], ['id-15', 'DeviceType'], ['id-15', 'DeviceInfo'], ['id-15', 'id-12'], ['id-15', 'id-16'], ['id-15', 'id-23'], ['id-15', 'id-27'], ['id-15', 'id-28'], ['id-15', 'id-29'], ['id-15', 'id-30'], ['id-15', 'id-31'], ['id-15', 'id-33'], ['id-15', 'id-34'], ['id-15', 'id-35'], ['id-15', 'id-36'], ['id-15', 'id-37'], ['id-15', 'id-38'], ['id-16', 'ProductCD'], ['id-16', 'card4'], ['id-16', 'card6'], ['id-16', 'P_emaildomain'], ['id-16', 'R_emaildomain'], ['id-16', 'M1'], ['id-16', 'M2'], ['id-16', 'M3'], ['id-16', 'M4'], ['id-16', 'M5'], ['id-16', 'M6'], ['id-16', 'M7'], ['id-16', 'M8'], ['id-16', 'M9'], ['id-16', 'id_12'], ['id-16', 'id_15'], ['id-16', 'id_16'], ['id-16', 'id_23'], ['id-16', 'id_27'], ['id-16', 'id_28'], ['id-16', 'id_29'], ['id-16', 'id_30'], ['id-16', 'id_31'], ['id-16', 'id_33'], ['id-16', 'id_34'], ['id-16', 'id_35'], ['id-16', 'id_36'], ['id-16', 'id_37'], ['id-16', 'id_38'], ['id-16', 'DeviceType'], ['id-16', 'DeviceInfo'], ['id-16', 'id-12'], ['id-16', 'id-15'], ['id-16', 'id-23'], ['id-16', 'id-27'], ['id-16', 'id-28'], ['id-16', 'id-29'], ['id-16', 'id-30'], ['id-16', 'id-31'], ['id-16', 'id-33'], ['id-16', 'id-34'], ['id-16', 'id-35'], ['id-16', 'id-36'], ['id-16', 'id-37'], ['id-16', 'id-38'], ['id-23', 'ProductCD'], ['id-23', 'card4'], ['id-23', 'card6'], ['id-23', 'P_emaildomain'], ['id-23', 'R_emaildomain'], ['id-23', 'M1'], ['id-23', 'M2'], ['id-23', 'M3'], ['id-23', 'M4'], ['id-23', 'M5'], ['id-23', 'M6'], ['id-23', 'M7'], ['id-23', 'M8'], ['id-23', 'M9'], ['id-23', 'id_12'], ['id-23', 'id_15'], ['id-23', 'id_16'], ['id-23', 'id_23'], ['id-23', 'id_27'], ['id-23', 'id_28'], ['id-23', 'id_29'], ['id-23', 'id_30'], ['id-23', 'id_31'], ['id-23', 'id_33'], ['id-23', 'id_34'], ['id-23', 'id_35'], ['id-23', 'id_36'], ['id-23', 'id_37'], ['id-23', 'id_38'], ['id-23', 'DeviceType'], ['id-23', 'DeviceInfo'], ['id-23', 'id-12'], ['id-23', 'id-15'], ['id-23', 'id-16'], ['id-23', 'id-27'], ['id-23', 'id-28'], ['id-23', 'id-29'], ['id-23', 'id-30'], ['id-23', 'id-31'], ['id-23', 'id-33'], ['id-23', 'id-34'], ['id-23', 'id-35'], ['id-23', 'id-36'], ['id-23', 'id-37'], ['id-23', 'id-38'], ['id-27', 'ProductCD'], ['id-27', 'card4'], ['id-27', 'card6'], ['id-27', 'P_emaildomain'], ['id-27', 'R_emaildomain'], ['id-27', 'M1'], ['id-27', 'M2'], ['id-27', 'M3'], ['id-27', 'M4'], ['id-27', 'M5'], ['id-27', 'M6'], ['id-27', 'M7'], ['id-27', 'M8'], ['id-27', 'M9'], ['id-27', 'id_12'], ['id-27', 'id_15'], ['id-27', 'id_16'], ['id-27', 'id_23'], ['id-27', 'id_27'], ['id-27', 'id_28'], ['id-27', 'id_29'], ['id-27', 'id_30'], ['id-27', 'id_31'], ['id-27', 'id_33'], ['id-27', 'id_34'], ['id-27', 'id_35'], ['id-27', 'id_36'], ['id-27', 'id_37'], ['id-27', 'id_38'], ['id-27', 'DeviceType'], ['id-27', 'DeviceInfo'], ['id-27', 'id-12'], ['id-27', 'id-15'], ['id-27', 'id-16'], ['id-27', 'id-23'], ['id-27', 'id-28'], ['id-27', 'id-29'], ['id-27', 'id-30'], ['id-27', 'id-31'], ['id-27', 'id-33'], ['id-27', 'id-34'], ['id-27', 'id-35'], ['id-27', 'id-36'], ['id-27', 'id-37'], ['id-27', 'id-38'], ['id-28', 'ProductCD'], ['id-28', 'card4'], ['id-28', 'card6'], ['id-28', 'P_emaildomain'], ['id-28', 'R_emaildomain'], ['id-28', 'M1'], ['id-28', 'M2'], ['id-28', 'M3'], ['id-28', 'M4'], ['id-28', 'M5'], ['id-28', 'M6'], ['id-28', 'M7'], ['id-28', 'M8'], ['id-28', 'M9'], ['id-28', 'id_12'], ['id-28', 'id_15'], ['id-28', 'id_16'], ['id-28', 'id_23'], ['id-28', 'id_27'], ['id-28', 'id_28'], ['id-28', 'id_29'], ['id-28', 'id_30'], ['id-28', 'id_31'], ['id-28', 'id_33'], ['id-28', 'id_34'], ['id-28', 'id_35'], ['id-28', 'id_36'], ['id-28', 'id_37'], ['id-28', 'id_38'], ['id-28', 'DeviceType'], ['id-28', 'DeviceInfo'], ['id-28', 'id-12'], ['id-28', 'id-15'], ['id-28', 'id-16'], ['id-28', 'id-23'], ['id-28', 'id-27'], ['id-28', 'id-29'], ['id-28', 'id-30'], ['id-28', 'id-31'], ['id-28', 'id-33'], ['id-28', 'id-34'], ['id-28', 'id-35'], ['id-28', 'id-36'], ['id-28', 'id-37'], ['id-28', 'id-38'], ['id-29', 'ProductCD'], ['id-29', 'card4'], ['id-29', 'card6'], ['id-29', 'P_emaildomain'], ['id-29', 'R_emaildomain'], ['id-29', 'M1'], ['id-29', 'M2'], ['id-29', 'M3'], ['id-29', 'M4'], ['id-29', 'M5'], ['id-29', 'M6'], ['id-29', 'M7'], ['id-29', 'M8'], ['id-29', 'M9'], ['id-29', 'id_12'], ['id-29', 'id_15'], ['id-29', 'id_16'], ['id-29', 'id_23'], ['id-29', 'id_27'], ['id-29', 'id_28'], ['id-29', 'id_29'], ['id-29', 'id_30'], ['id-29', 'id_31'], ['id-29', 'id_33'], ['id-29', 'id_34'], ['id-29', 'id_35'], ['id-29', 'id_36'], ['id-29', 'id_37'], ['id-29', 'id_38'], ['id-29', 'DeviceType'], ['id-29', 'DeviceInfo'], ['id-29', 'id-12'], ['id-29', 'id-15'], ['id-29', 'id-16'], ['id-29', 'id-23'], ['id-29', 'id-27'], ['id-29', 'id-28'], ['id-29', 'id-30'], ['id-29', 'id-31'], ['id-29', 'id-33'], ['id-29', 'id-34'], ['id-29', 'id-35'], ['id-29', 'id-36'], ['id-29', 'id-37'], ['id-29', 'id-38'], ['id-30', 'ProductCD'], ['id-30', 'card4'], ['id-30', 'card6'], ['id-30', 'P_emaildomain'], ['id-30', 'R_emaildomain'], ['id-30', 'M1'], ['id-30', 'M2'], ['id-30', 'M3'], ['id-30', 'M4'], ['id-30', 'M5'], ['id-30', 'M6'], ['id-30', 'M7'], ['id-30', 'M8'], ['id-30', 'M9'], ['id-30', 'id_12'], ['id-30', 'id_15'], ['id-30', 'id_16'], ['id-30', 'id_23'], ['id-30', 'id_27'], ['id-30', 'id_28'], ['id-30', 'id_29'], ['id-30', 'id_30'], ['id-30', 'id_31'], ['id-30', 'id_33'], ['id-30', 'id_34'], ['id-30', 'id_35'], ['id-30', 'id_36'], ['id-30', 'id_37'], ['id-30', 'id_38'], ['id-30', 'DeviceType'], ['id-30', 'DeviceInfo'], ['id-30', 'id-12'], ['id-30', 'id-15'], ['id-30', 'id-16'], ['id-30', 'id-23'], ['id-30', 'id-27'], ['id-30', 'id-28'], ['id-30', 'id-29'], ['id-30', 'id-31'], ['id-30', 'id-33'], ['id-30', 'id-34'], ['id-30', 'id-35'], ['id-30', 'id-36'], ['id-30', 'id-37'], ['id-30', 'id-38'], ['id-31', 'ProductCD'], ['id-31', 'card4'], ['id-31', 'card6'], ['id-31', 'P_emaildomain'], ['id-31', 'R_emaildomain'], ['id-31', 'M1'], ['id-31', 'M2'], ['id-31', 'M3'], ['id-31', 'M4'], ['id-31', 'M5'], ['id-31', 'M6'], ['id-31', 'M7'], ['id-31', 'M8'], ['id-31', 'M9'], ['id-31', 'id_12'], ['id-31', 'id_15'], ['id-31', 'id_16'], ['id-31', 'id_23'], ['id-31', 'id_27'], ['id-31', 'id_28'], ['id-31', 'id_29'], ['id-31', 'id_30'], ['id-31', 'id_31'], ['id-31', 'id_33'], ['id-31', 'id_34'], ['id-31', 'id_35'], ['id-31', 'id_36'], ['id-31', 'id_37'], ['id-31', 'id_38'], ['id-31', 'DeviceType'], ['id-31', 'DeviceInfo'], ['id-31', 'id-12'], ['id-31', 'id-15'], ['id-31', 'id-16'], ['id-31', 'id-23'], ['id-31', 'id-27'], ['id-31', 'id-28'], ['id-31', 'id-29'], ['id-31', 'id-30'], ['id-31', 'id-33'], ['id-31', 'id-34'], ['id-31', 'id-35'], ['id-31', 'id-36'], ['id-31', 'id-37'], ['id-31', 'id-38'], ['id-33', 'ProductCD'], ['id-33', 'card4'], ['id-33', 'card6'], ['id-33', 'P_emaildomain'], ['id-33', 'R_emaildomain'], ['id-33', 'M1'], ['id-33', 'M2'], ['id-33', 'M3'], ['id-33', 'M4'], ['id-33', 'M5'], ['id-33', 'M6'], ['id-33', 'M7'], ['id-33', 'M8'], ['id-33', 'M9'], ['id-33', 'id_12'], ['id-33', 'id_15'], ['id-33', 'id_16'], ['id-33', 'id_23'], ['id-33', 'id_27'], ['id-33', 'id_28'], ['id-33', 'id_29'], ['id-33', 'id_30'], ['id-33', 'id_31'], ['id-33', 'id_33'], ['id-33', 'id_34'], ['id-33', 'id_35'], ['id-33', 'id_36'], ['id-33', 'id_37'], ['id-33', 'id_38'], ['id-33', 'DeviceType'], ['id-33', 'DeviceInfo'], ['id-33', 'id-12'], ['id-33', 'id-15'], ['id-33', 'id-16'], ['id-33', 'id-23'], ['id-33', 'id-27'], ['id-33', 'id-28'], ['id-33', 'id-29'], ['id-33', 'id-30'], ['id-33', 'id-31'], ['id-33', 'id-34'], ['id-33', 'id-35'], ['id-33', 'id-36'], ['id-33', 'id-37'], ['id-33', 'id-38'], ['id-34', 'ProductCD'], ['id-34', 'card4'], ['id-34', 'card6'], ['id-34', 'P_emaildomain'], ['id-34', 'R_emaildomain'], ['id-34', 'M1'], ['id-34', 'M2'], ['id-34', 'M3'], ['id-34', 'M4'], ['id-34', 'M5'], ['id-34', 'M6'], ['id-34', 'M7'], ['id-34', 'M8'], ['id-34', 'M9'], ['id-34', 'id_12'], ['id-34', 'id_15'], ['id-34', 'id_16'], ['id-34', 'id_23'], ['id-34', 'id_27'], ['id-34', 'id_28'], ['id-34', 'id_29'], ['id-34', 'id_30'], ['id-34', 'id_31'], ['id-34', 'id_33'], ['id-34', 'id_34'], ['id-34', 'id_35'], ['id-34', 'id_36'], ['id-34', 'id_37'], ['id-34', 'id_38'], ['id-34', 'DeviceType'], ['id-34', 'DeviceInfo'], ['id-34', 'id-12'], ['id-34', 'id-15'], ['id-34', 'id-16'], ['id-34', 'id-23'], ['id-34', 'id-27'], ['id-34', 'id-28'], ['id-34', 'id-29'], ['id-34', 'id-30'], ['id-34', 'id-31'], ['id-34', 'id-33'], ['id-34', 'id-35'], ['id-34', 'id-36'], ['id-34', 'id-37'], ['id-34', 'id-38'], ['id-35', 'ProductCD'], ['id-35', 'card4'], ['id-35', 'card6'], ['id-35', 'P_emaildomain'], ['id-35', 'R_emaildomain'], ['id-35', 'M1'], ['id-35', 'M2'], ['id-35', 'M3'], ['id-35', 'M4'], ['id-35', 'M5'], ['id-35', 'M6'], ['id-35', 'M7'], ['id-35', 'M8'], ['id-35', 'M9'], ['id-35', 'id_12'], ['id-35', 'id_15'], ['id-35', 'id_16'], ['id-35', 'id_23'], ['id-35', 'id_27'], ['id-35', 'id_28'], ['id-35', 'id_29'], ['id-35', 'id_30'], ['id-35', 'id_31'], ['id-35', 'id_33'], ['id-35', 'id_34'], ['id-35', 'id_35'], ['id-35', 'id_36'], ['id-35', 'id_37'], ['id-35', 'id_38'], ['id-35', 'DeviceType'], ['id-35', 'DeviceInfo'], ['id-35', 'id-12'], ['id-35', 'id-15'], ['id-35', 'id-16'], ['id-35', 'id-23'], ['id-35', 'id-27'], ['id-35', 'id-28'], ['id-35', 'id-29'], ['id-35', 'id-30'], ['id-35', 'id-31'], ['id-35', 'id-33'], ['id-35', 'id-34'], ['id-35', 'id-36'], ['id-35', 'id-37'], ['id-35', 'id-38'], ['id-36', 'ProductCD'], ['id-36', 'card4'], ['id-36', 'card6'], ['id-36', 'P_emaildomain'], ['id-36', 'R_emaildomain'], ['id-36', 'M1'], ['id-36', 'M2'], ['id-36', 'M3'], ['id-36', 'M4'], ['id-36', 'M5'], ['id-36', 'M6'], ['id-36', 'M7'], ['id-36', 'M8'], ['id-36', 'M9'], ['id-36', 'id_12'], ['id-36', 'id_15'], ['id-36', 'id_16'], ['id-36', 'id_23'], ['id-36', 'id_27'], ['id-36', 'id_28'], ['id-36', 'id_29'], ['id-36', 'id_30'], ['id-36', 'id_31'], ['id-36', 'id_33'], ['id-36', 'id_34'], ['id-36', 'id_35'], ['id-36', 'id_36'], ['id-36', 'id_37'], ['id-36', 'id_38'], ['id-36', 'DeviceType'], ['id-36', 'DeviceInfo'], ['id-36', 'id-12'], ['id-36', 'id-15'], ['id-36', 'id-16'], ['id-36', 'id-23'], ['id-36', 'id-27'], ['id-36', 'id-28'], ['id-36', 'id-29'], ['id-36', 'id-30'], ['id-36', 'id-31'], ['id-36', 'id-33'], ['id-36', 'id-34'], ['id-36', 'id-35'], ['id-36', 'id-37'], ['id-36', 'id-38'], ['id-37', 'ProductCD'], ['id-37', 'card4'], ['id-37', 'card6'], ['id-37', 'P_emaildomain'], ['id-37', 'R_emaildomain'], ['id-37', 'M1'], ['id-37', 'M2'], ['id-37', 'M3'], ['id-37', 'M4'], ['id-37', 'M5'], ['id-37', 'M6'], ['id-37', 'M7'], ['id-37', 'M8'], ['id-37', 'M9'], ['id-37', 'id_12'], ['id-37', 'id_15'], ['id-37', 'id_16'], ['id-37', 'id_23'], ['id-37', 'id_27'], ['id-37', 'id_28'], ['id-37', 'id_29'], ['id-37', 'id_30'], ['id-37', 'id_31'], ['id-37', 'id_33'], ['id-37', 'id_34'], ['id-37', 'id_35'], ['id-37', 'id_36'], ['id-37', 'id_37'], ['id-37', 'id_38'], ['id-37', 'DeviceType'], ['id-37', 'DeviceInfo'], ['id-37', 'id-12'], ['id-37', 'id-15'], ['id-37', 'id-16'], ['id-37', 'id-23'], ['id-37', 'id-27'], ['id-37', 'id-28'], ['id-37', 'id-29'], ['id-37', 'id-30'], ['id-37', 'id-31'], ['id-37', 'id-33'], ['id-37', 'id-34'], ['id-37', 'id-35'], ['id-37', 'id-36'], ['id-37', 'id-38'], ['id-38', 'ProductCD'], ['id-38', 'card4'], ['id-38', 'card6'], ['id-38', 'P_emaildomain'], ['id-38', 'R_emaildomain'], ['id-38', 'M1'], ['id-38', 'M2'], ['id-38', 'M3'], ['id-38', 'M4'], ['id-38', 'M5'], ['id-38', 'M6'], ['id-38', 'M7'], ['id-38', 'M8'], ['id-38', 'M9'], ['id-38', 'id_12'], ['id-38', 'id_15'], ['id-38', 'id_16'], ['id-38', 'id_23'], ['id-38', 'id_27'], ['id-38', 'id_28'], ['id-38', 'id_29'], ['id-38', 'id_30'], ['id-38', 'id_31'], ['id-38', 'id_33'], ['id-38', 'id_34'], ['id-38', 'id_35'], ['id-38', 'id_36'], ['id-38', 'id_37'], ['id-38', 'id_38'], ['id-38', 'DeviceType'], ['id-38', 'DeviceInfo'], ['id-38', 'id-12'], ['id-38', 'id-15'], ['id-38', 'id-16'], ['id-38', 'id-23'], ['id-38', 'id-27'], ['id-38', 'id-28'], ['id-38', 'id-29'], ['id-38', 'id-30'], ['id-38', 'id-31'], ['id-38', 'id-33'], ['id-38', 'id-34'], ['id-38', 'id-35'], ['id-38', 'id-36'], ['id-38', 'id-37'], ['ProductCD'], ['card4'], ['card6'], ['P_emaildomain'], ['R_emaildomain'], ['M1'], ['M2'], ['M3'], ['M4'], ['M5'], ['M6'], ['M7'], ['M8'], ['M9'], ['id_12'], ['id_15'], ['id_16'], ['id_23'], ['id_27'], ['id_28'], ['id_29'], ['id_30'], ['id_31'], ['id_33'], ['id_34'], ['id_35'], ['id_36'], ['id_37'], ['id_38'], ['DeviceType'], ['DeviceInfo'], ['id-12'], ['id-15'], ['id-16'], ['id-23'], ['id-27'], ['id-28'], ['id-29'], ['id-30'], ['id-31'], ['id-33'], ['id-34'], ['id-35'], ['id-36'], ['id-37'], ['id-38']]\n"
     ]
    }
   ],
   "source": [
    "log(featureCount.get_ops())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "14583bfa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.814720Z",
     "start_time": "2021-09-02T07:47:50.810974Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2116"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(featureCount.get_ops())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "dbb52c0d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.818435Z",
     "start_time": "2021-09-02T07:47:50.816215Z"
    }
   },
   "outputs": [],
   "source": [
    "# 手动修改配置\n",
    "# featureCount.set_keys([[],[]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "ee2d6961",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.822110Z",
     "start_time": "2021-09-02T07:47:50.819974Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# FE_count = featureCount.transform(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "796ae1a1",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.825931Z",
     "start_time": "2021-09-02T07:47:50.823675Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# FE_count.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eccc5f25",
   "metadata": {},
   "source": [
    "## 特征合并"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "4413653e",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.829838Z",
     "start_time": "2021-09-02T07:47:50.827473Z"
    }
   },
   "outputs": [],
   "source": [
    "from autox.process_data import feature_combination"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "0a0fcdaa",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.833913Z",
     "start_time": "2021-09-02T07:47:50.831363Z"
    }
   },
   "outputs": [],
   "source": [
    "df_list = [df]\n",
    "# df_list = [df, FE_count, FE_stat, FE_rank]\n",
    "\n",
    "FE_all = feature_combination(df_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "16e023b6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.839088Z",
     "start_time": "2021-09-02T07:47:50.835645Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1097231, 472)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "FE_all.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8c75570a",
   "metadata": {},
   "source": [
    "## train和test数据切分"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "92cb1875",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.846399Z",
     "start_time": "2021-09-02T07:47:50.844122Z"
    }
   },
   "outputs": [],
   "source": [
    "from autox.process_data import train_test_divide"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "7a90d0fc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.851290Z",
     "start_time": "2021-09-02T07:47:50.847849Z"
    }
   },
   "outputs": [],
   "source": [
    "train_length = autox.info_['shape_of_train']\n",
    "train, test = train_test_divide(FE_all, train_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "f8dea58b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.856248Z",
     "start_time": "2021-09-02T07:47:50.852693Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((590540, 472), (506691, 472))"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape, test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7a71e8dd",
   "metadata": {},
   "source": [
    "## 特征过滤"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "2df700af",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:47:50.859904Z",
     "start_time": "2021-09-02T07:47:50.857684Z"
    }
   },
   "outputs": [],
   "source": [
    "from autox.process_data import feature_filter"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "2d859d07",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:54:46.472935Z",
     "start_time": "2021-09-02T07:47:50.861402Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 426/426 [06:26<00:00,  1.10it/s]\n",
      "   INFO ->  filtered features: ['TransactionID', 'isFraud', 'TransactionDT', 'id_01', 'id_02', 'id_03', 'id_04', 'id_05', 'id_06', 'id_07', 'id_08', 'id_09', 'id_10', 'id_11', 'id_13', 'id_14', 'id_17', 'id_18', 'id_19', 'id_20', 'id_21', 'id_22', 'id_24', 'id_25', 'id_26', 'id_32']\n"
     ]
    }
   ],
   "source": [
    "used_features = feature_filter(train, test, id_, target)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2bd606eb",
   "metadata": {},
   "source": [
    "## 查看最终使用的特征"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "b606d37a",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:54:46.480410Z",
     "start_time": "2021-09-02T07:54:46.475351Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "400"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(used_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "b872de35",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T07:54:46.492248Z",
     "start_time": "2021-09-02T07:54:46.482047Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['TransactionAmt',\n",
       " 'card1',\n",
       " 'card2',\n",
       " 'card3',\n",
       " 'card5',\n",
       " 'addr1',\n",
       " 'addr2',\n",
       " 'dist1',\n",
       " 'dist2',\n",
       " 'C1',\n",
       " 'C2',\n",
       " 'C3',\n",
       " 'C4',\n",
       " 'C5',\n",
       " 'C6',\n",
       " 'C7',\n",
       " 'C8',\n",
       " 'C9',\n",
       " 'C10',\n",
       " 'C11',\n",
       " 'C12',\n",
       " 'C13',\n",
       " 'C14',\n",
       " 'D1',\n",
       " 'D2',\n",
       " 'D3',\n",
       " 'D4',\n",
       " 'D5',\n",
       " 'D6',\n",
       " 'D7',\n",
       " 'D8',\n",
       " 'D9',\n",
       " 'D10',\n",
       " 'D11',\n",
       " 'D12',\n",
       " 'D13',\n",
       " 'D14',\n",
       " 'D15',\n",
       " 'V1',\n",
       " 'V2',\n",
       " 'V3',\n",
       " 'V4',\n",
       " 'V5',\n",
       " 'V6',\n",
       " 'V7',\n",
       " 'V8',\n",
       " 'V9',\n",
       " 'V10',\n",
       " 'V11',\n",
       " 'V12',\n",
       " 'V13',\n",
       " 'V14',\n",
       " 'V15',\n",
       " 'V16',\n",
       " 'V17',\n",
       " 'V18',\n",
       " 'V19',\n",
       " 'V20',\n",
       " 'V21',\n",
       " 'V22',\n",
       " 'V23',\n",
       " 'V24',\n",
       " 'V25',\n",
       " 'V26',\n",
       " 'V27',\n",
       " 'V28',\n",
       " 'V29',\n",
       " 'V30',\n",
       " 'V31',\n",
       " 'V32',\n",
       " 'V33',\n",
       " 'V34',\n",
       " 'V35',\n",
       " 'V36',\n",
       " 'V37',\n",
       " 'V38',\n",
       " 'V39',\n",
       " 'V40',\n",
       " 'V41',\n",
       " 'V42',\n",
       " 'V43',\n",
       " 'V44',\n",
       " 'V45',\n",
       " 'V46',\n",
       " 'V47',\n",
       " 'V48',\n",
       " 'V49',\n",
       " 'V50',\n",
       " 'V51',\n",
       " 'V52',\n",
       " 'V53',\n",
       " 'V54',\n",
       " 'V55',\n",
       " 'V56',\n",
       " 'V57',\n",
       " 'V58',\n",
       " 'V59',\n",
       " 'V60',\n",
       " 'V61',\n",
       " 'V62',\n",
       " 'V63',\n",
       " 'V64',\n",
       " 'V65',\n",
       " 'V66',\n",
       " 'V67',\n",
       " 'V68',\n",
       " 'V69',\n",
       " 'V70',\n",
       " 'V71',\n",
       " 'V72',\n",
       " 'V73',\n",
       " 'V74',\n",
       " 'V75',\n",
       " 'V76',\n",
       " 'V77',\n",
       " 'V78',\n",
       " 'V79',\n",
       " 'V80',\n",
       " 'V81',\n",
       " 'V82',\n",
       " 'V83',\n",
       " 'V84',\n",
       " 'V85',\n",
       " 'V86',\n",
       " 'V87',\n",
       " 'V88',\n",
       " 'V89',\n",
       " 'V90',\n",
       " 'V91',\n",
       " 'V92',\n",
       " 'V93',\n",
       " 'V94',\n",
       " 'V95',\n",
       " 'V96',\n",
       " 'V97',\n",
       " 'V98',\n",
       " 'V99',\n",
       " 'V100',\n",
       " 'V101',\n",
       " 'V102',\n",
       " 'V103',\n",
       " 'V104',\n",
       " 'V105',\n",
       " 'V106',\n",
       " 'V107',\n",
       " 'V108',\n",
       " 'V109',\n",
       " 'V110',\n",
       " 'V111',\n",
       " 'V112',\n",
       " 'V113',\n",
       " 'V114',\n",
       " 'V115',\n",
       " 'V116',\n",
       " 'V117',\n",
       " 'V118',\n",
       " 'V119',\n",
       " 'V120',\n",
       " 'V121',\n",
       " 'V122',\n",
       " 'V123',\n",
       " 'V124',\n",
       " 'V125',\n",
       " 'V126',\n",
       " 'V127',\n",
       " 'V128',\n",
       " 'V129',\n",
       " 'V130',\n",
       " 'V131',\n",
       " 'V132',\n",
       " 'V133',\n",
       " 'V134',\n",
       " 'V135',\n",
       " 'V136',\n",
       " 'V137',\n",
       " 'V138',\n",
       " 'V139',\n",
       " 'V140',\n",
       " 'V141',\n",
       " 'V142',\n",
       " 'V143',\n",
       " 'V144',\n",
       " 'V145',\n",
       " 'V146',\n",
       " 'V147',\n",
       " 'V148',\n",
       " 'V149',\n",
       " 'V150',\n",
       " 'V151',\n",
       " 'V152',\n",
       " 'V153',\n",
       " 'V154',\n",
       " 'V155',\n",
       " 'V156',\n",
       " 'V157',\n",
       " 'V158',\n",
       " 'V159',\n",
       " 'V160',\n",
       " 'V161',\n",
       " 'V162',\n",
       " 'V163',\n",
       " 'V164',\n",
       " 'V165',\n",
       " 'V166',\n",
       " 'V167',\n",
       " 'V168',\n",
       " 'V169',\n",
       " 'V170',\n",
       " 'V171',\n",
       " 'V172',\n",
       " 'V173',\n",
       " 'V174',\n",
       " 'V175',\n",
       " 'V176',\n",
       " 'V177',\n",
       " 'V178',\n",
       " 'V179',\n",
       " 'V180',\n",
       " 'V181',\n",
       " 'V182',\n",
       " 'V183',\n",
       " 'V184',\n",
       " 'V185',\n",
       " 'V186',\n",
       " 'V187',\n",
       " 'V188',\n",
       " 'V189',\n",
       " 'V190',\n",
       " 'V191',\n",
       " 'V192',\n",
       " 'V193',\n",
       " 'V194',\n",
       " 'V195',\n",
       " 'V196',\n",
       " 'V197',\n",
       " 'V198',\n",
       " 'V199',\n",
       " 'V200',\n",
       " 'V201',\n",
       " 'V202',\n",
       " 'V203',\n",
       " 'V204',\n",
       " 'V205',\n",
       " 'V206',\n",
       " 'V207',\n",
       " 'V208',\n",
       " 'V209',\n",
       " 'V210',\n",
       " 'V211',\n",
       " 'V212',\n",
       " 'V213',\n",
       " 'V214',\n",
       " 'V215',\n",
       " 'V216',\n",
       " 'V217',\n",
       " 'V218',\n",
       " 'V219',\n",
       " 'V220',\n",
       " 'V221',\n",
       " 'V222',\n",
       " 'V223',\n",
       " 'V224',\n",
       " 'V225',\n",
       " 'V226',\n",
       " 'V227',\n",
       " 'V228',\n",
       " 'V229',\n",
       " 'V230',\n",
       " 'V231',\n",
       " 'V232',\n",
       " 'V233',\n",
       " 'V234',\n",
       " 'V235',\n",
       " 'V236',\n",
       " 'V237',\n",
       " 'V238',\n",
       " 'V239',\n",
       " 'V240',\n",
       " 'V241',\n",
       " 'V242',\n",
       " 'V243',\n",
       " 'V244',\n",
       " 'V245',\n",
       " 'V246',\n",
       " 'V247',\n",
       " 'V248',\n",
       " 'V249',\n",
       " 'V250',\n",
       " 'V251',\n",
       " 'V252',\n",
       " 'V253',\n",
       " 'V254',\n",
       " 'V255',\n",
       " 'V256',\n",
       " 'V257',\n",
       " 'V258',\n",
       " 'V259',\n",
       " 'V260',\n",
       " 'V261',\n",
       " 'V262',\n",
       " 'V263',\n",
       " 'V264',\n",
       " 'V265',\n",
       " 'V266',\n",
       " 'V267',\n",
       " 'V268',\n",
       " 'V269',\n",
       " 'V270',\n",
       " 'V271',\n",
       " 'V272',\n",
       " 'V273',\n",
       " 'V274',\n",
       " 'V275',\n",
       " 'V276',\n",
       " 'V277',\n",
       " 'V278',\n",
       " 'V279',\n",
       " 'V280',\n",
       " 'V281',\n",
       " 'V282',\n",
       " 'V283',\n",
       " 'V284',\n",
       " 'V285',\n",
       " 'V286',\n",
       " 'V287',\n",
       " 'V288',\n",
       " 'V289',\n",
       " 'V290',\n",
       " 'V291',\n",
       " 'V292',\n",
       " 'V293',\n",
       " 'V294',\n",
       " 'V295',\n",
       " 'V296',\n",
       " 'V297',\n",
       " 'V298',\n",
       " 'V299',\n",
       " 'V300',\n",
       " 'V301',\n",
       " 'V302',\n",
       " 'V303',\n",
       " 'V304',\n",
       " 'V305',\n",
       " 'V306',\n",
       " 'V307',\n",
       " 'V308',\n",
       " 'V309',\n",
       " 'V310',\n",
       " 'V311',\n",
       " 'V312',\n",
       " 'V313',\n",
       " 'V314',\n",
       " 'V315',\n",
       " 'V316',\n",
       " 'V317',\n",
       " 'V318',\n",
       " 'V319',\n",
       " 'V320',\n",
       " 'V321',\n",
       " 'V322',\n",
       " 'V323',\n",
       " 'V324',\n",
       " 'V325',\n",
       " 'V326',\n",
       " 'V327',\n",
       " 'V328',\n",
       " 'V329',\n",
       " 'V330',\n",
       " 'V331',\n",
       " 'V332',\n",
       " 'V333',\n",
       " 'V334',\n",
       " 'V335',\n",
       " 'V336',\n",
       " 'V337',\n",
       " 'V338',\n",
       " 'V339',\n",
       " 'id-01',\n",
       " 'id-02',\n",
       " 'id-03',\n",
       " 'id-04',\n",
       " 'id-05',\n",
       " 'id-06',\n",
       " 'id-07',\n",
       " 'id-08',\n",
       " 'id-09',\n",
       " 'id-10',\n",
       " 'id-11',\n",
       " 'id-13',\n",
       " 'id-14',\n",
       " 'id-17',\n",
       " 'id-18',\n",
       " 'id-19',\n",
       " 'id-20',\n",
       " 'id-21',\n",
       " 'id-22',\n",
       " 'id-24',\n",
       " 'id-25',\n",
       " 'id-26',\n",
       " 'id-32']"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "used_features"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "33ad464c",
   "metadata": {},
   "source": [
    "## 模型训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "28462ed6",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:23:41.258124Z",
     "start_time": "2021-09-02T07:57:07.092539Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "   INFO ->  (590540, 400)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training on fold 1\n",
      "Training until validation scores don't improve for 150 rounds\n",
      "[100]\ttraining's auc: 0.870914\tvalid_1's auc: 0.858531\n",
      "[200]\ttraining's auc: 0.885236\tvalid_1's auc: 0.869557\n",
      "[300]\ttraining's auc: 0.893103\tvalid_1's auc: 0.877265\n",
      "[400]\ttraining's auc: 0.898723\tvalid_1's auc: 0.882655\n",
      "[500]\ttraining's auc: 0.902938\tvalid_1's auc: 0.886648\n",
      "[600]\ttraining's auc: 0.906642\tvalid_1's auc: 0.890071\n",
      "[700]\ttraining's auc: 0.909995\tvalid_1's auc: 0.893169\n",
      "[800]\ttraining's auc: 0.912443\tvalid_1's auc: 0.89551\n",
      "[900]\ttraining's auc: 0.914933\tvalid_1's auc: 0.897679\n",
      "[1000]\ttraining's auc: 0.917149\tvalid_1's auc: 0.899653\n",
      "[1100]\ttraining's auc: 0.9194\tvalid_1's auc: 0.901604\n",
      "[1200]\ttraining's auc: 0.921115\tvalid_1's auc: 0.902974\n",
      "[1300]\ttraining's auc: 0.923097\tvalid_1's auc: 0.904674\n",
      "[1400]\ttraining's auc: 0.924758\tvalid_1's auc: 0.906087\n",
      "[1500]\ttraining's auc: 0.926161\tvalid_1's auc: 0.907344\n",
      "[1600]\ttraining's auc: 0.927565\tvalid_1's auc: 0.908556\n",
      "[1700]\ttraining's auc: 0.929065\tvalid_1's auc: 0.909871\n",
      "[1800]\ttraining's auc: 0.930297\tvalid_1's auc: 0.910886\n",
      "[1900]\ttraining's auc: 0.931434\tvalid_1's auc: 0.9118\n",
      "[2000]\ttraining's auc: 0.93238\tvalid_1's auc: 0.912668\n",
      "[2100]\ttraining's auc: 0.933633\tvalid_1's auc: 0.913714\n",
      "[2200]\ttraining's auc: 0.93466\tvalid_1's auc: 0.914672\n",
      "[2300]\ttraining's auc: 0.935662\tvalid_1's auc: 0.915542\n",
      "[2400]\ttraining's auc: 0.936669\tvalid_1's auc: 0.916407\n",
      "[2500]\ttraining's auc: 0.937707\tvalid_1's auc: 0.917382\n",
      "[2600]\ttraining's auc: 0.938726\tvalid_1's auc: 0.918235\n",
      "[2700]\ttraining's auc: 0.939709\tvalid_1's auc: 0.919081\n",
      "[2800]\ttraining's auc: 0.940551\tvalid_1's auc: 0.919761\n",
      "[2900]\ttraining's auc: 0.941316\tvalid_1's auc: 0.920449\n",
      "[3000]\ttraining's auc: 0.942065\tvalid_1's auc: 0.921131\n",
      "[3100]\ttraining's auc: 0.942878\tvalid_1's auc: 0.921867\n",
      "[3200]\ttraining's auc: 0.94358\tvalid_1's auc: 0.922461\n",
      "[3300]\ttraining's auc: 0.944218\tvalid_1's auc: 0.922998\n",
      "[3400]\ttraining's auc: 0.944917\tvalid_1's auc: 0.923581\n",
      "[3500]\ttraining's auc: 0.945532\tvalid_1's auc: 0.924084\n",
      "[3600]\ttraining's auc: 0.946272\tvalid_1's auc: 0.924653\n",
      "[3700]\ttraining's auc: 0.946912\tvalid_1's auc: 0.925156\n",
      "[3800]\ttraining's auc: 0.94782\tvalid_1's auc: 0.925876\n",
      "[3900]\ttraining's auc: 0.948454\tvalid_1's auc: 0.926363\n",
      "[4000]\ttraining's auc: 0.949145\tvalid_1's auc: 0.926932\n",
      "[4100]\ttraining's auc: 0.94974\tvalid_1's auc: 0.927444\n",
      "[4200]\ttraining's auc: 0.950333\tvalid_1's auc: 0.927922\n",
      "[4300]\ttraining's auc: 0.950995\tvalid_1's auc: 0.92851\n",
      "[4400]\ttraining's auc: 0.951558\tvalid_1's auc: 0.928884\n",
      "[4500]\ttraining's auc: 0.952138\tvalid_1's auc: 0.929438\n",
      "[4600]\ttraining's auc: 0.952745\tvalid_1's auc: 0.929914\n",
      "[4700]\ttraining's auc: 0.953279\tvalid_1's auc: 0.930379\n",
      "[4800]\ttraining's auc: 0.953851\tvalid_1's auc: 0.930889\n",
      "[4900]\ttraining's auc: 0.954446\tvalid_1's auc: 0.931356\n",
      "[5000]\ttraining's auc: 0.954955\tvalid_1's auc: 0.931796\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[5000]\ttraining's auc: 0.954955\tvalid_1's auc: 0.931796\n",
      "AUC: 0.9317958291973454\n",
      "Fold 1 finished in 0:05:04.756749\n",
      "Training on fold 2\n",
      "Training until validation scores don't improve for 150 rounds\n",
      "[100]\ttraining's auc: 0.869572\tvalid_1's auc: 0.873341\n",
      "[200]\ttraining's auc: 0.88228\tvalid_1's auc: 0.885611\n",
      "[300]\ttraining's auc: 0.890358\tvalid_1's auc: 0.892279\n",
      "[400]\ttraining's auc: 0.895499\tvalid_1's auc: 0.896569\n",
      "[500]\ttraining's auc: 0.900228\tvalid_1's auc: 0.900467\n",
      "[600]\ttraining's auc: 0.903907\tvalid_1's auc: 0.903179\n",
      "[700]\ttraining's auc: 0.907087\tvalid_1's auc: 0.905517\n",
      "[800]\ttraining's auc: 0.909474\tvalid_1's auc: 0.907336\n",
      "[900]\ttraining's auc: 0.91151\tvalid_1's auc: 0.90876\n",
      "[1000]\ttraining's auc: 0.913653\tvalid_1's auc: 0.910261\n",
      "[1100]\ttraining's auc: 0.915768\tvalid_1's auc: 0.911799\n",
      "[1200]\ttraining's auc: 0.917874\tvalid_1's auc: 0.913471\n",
      "[1300]\ttraining's auc: 0.919814\tvalid_1's auc: 0.914837\n",
      "[1400]\ttraining's auc: 0.921711\tvalid_1's auc: 0.916204\n",
      "[1500]\ttraining's auc: 0.923334\tvalid_1's auc: 0.917226\n",
      "[1600]\ttraining's auc: 0.924933\tvalid_1's auc: 0.918433\n",
      "[1700]\ttraining's auc: 0.926439\tvalid_1's auc: 0.919453\n",
      "[1800]\ttraining's auc: 0.928042\tvalid_1's auc: 0.920506\n",
      "[1900]\ttraining's auc: 0.929262\tvalid_1's auc: 0.921356\n",
      "[2000]\ttraining's auc: 0.930579\tvalid_1's auc: 0.922318\n",
      "[2100]\ttraining's auc: 0.931624\tvalid_1's auc: 0.922995\n",
      "[2200]\ttraining's auc: 0.932762\tvalid_1's auc: 0.923777\n",
      "[2300]\ttraining's auc: 0.933968\tvalid_1's auc: 0.924564\n",
      "[2400]\ttraining's auc: 0.935187\tvalid_1's auc: 0.925455\n",
      "[2500]\ttraining's auc: 0.936236\tvalid_1's auc: 0.926103\n",
      "[2600]\ttraining's auc: 0.937075\tvalid_1's auc: 0.926638\n",
      "[2700]\ttraining's auc: 0.93793\tvalid_1's auc: 0.927179\n",
      "[2800]\ttraining's auc: 0.938993\tvalid_1's auc: 0.927903\n",
      "[2900]\ttraining's auc: 0.939871\tvalid_1's auc: 0.928445\n",
      "[3000]\ttraining's auc: 0.940727\tvalid_1's auc: 0.929044\n",
      "[3100]\ttraining's auc: 0.941454\tvalid_1's auc: 0.929597\n",
      "[3200]\ttraining's auc: 0.942126\tvalid_1's auc: 0.930027\n",
      "[3300]\ttraining's auc: 0.942942\tvalid_1's auc: 0.930528\n",
      "[3400]\ttraining's auc: 0.943724\tvalid_1's auc: 0.931057\n",
      "[3500]\ttraining's auc: 0.944539\tvalid_1's auc: 0.931644\n",
      "[3600]\ttraining's auc: 0.945307\tvalid_1's auc: 0.932146\n",
      "[3700]\ttraining's auc: 0.946091\tvalid_1's auc: 0.932645\n",
      "[3800]\ttraining's auc: 0.946731\tvalid_1's auc: 0.933097\n",
      "[3900]\ttraining's auc: 0.947384\tvalid_1's auc: 0.933533\n",
      "[4000]\ttraining's auc: 0.948055\tvalid_1's auc: 0.93399\n",
      "[4100]\ttraining's auc: 0.948637\tvalid_1's auc: 0.934374\n",
      "[4200]\ttraining's auc: 0.949157\tvalid_1's auc: 0.934715\n",
      "[4300]\ttraining's auc: 0.949782\tvalid_1's auc: 0.935193\n",
      "[4400]\ttraining's auc: 0.950342\tvalid_1's auc: 0.935576\n",
      "[4500]\ttraining's auc: 0.950996\tvalid_1's auc: 0.935957\n",
      "[4600]\ttraining's auc: 0.951596\tvalid_1's auc: 0.936332\n",
      "[4700]\ttraining's auc: 0.952171\tvalid_1's auc: 0.936696\n",
      "[4800]\ttraining's auc: 0.952728\tvalid_1's auc: 0.937076\n",
      "[4900]\ttraining's auc: 0.953214\tvalid_1's auc: 0.937421\n",
      "[5000]\ttraining's auc: 0.953759\tvalid_1's auc: 0.937737\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[5000]\ttraining's auc: 0.953759\tvalid_1's auc: 0.937737\n",
      "AUC: 0.9377373848499838\n",
      "Fold 2 finished in 0:04:59.527505\n",
      "Training on fold 3\n",
      "Training until validation scores don't improve for 150 rounds\n",
      "[100]\ttraining's auc: 0.870509\tvalid_1's auc: 0.864051\n",
      "[200]\ttraining's auc: 0.883945\tvalid_1's auc: 0.875257\n",
      "[300]\ttraining's auc: 0.892224\tvalid_1's auc: 0.882\n",
      "[400]\ttraining's auc: 0.897681\tvalid_1's auc: 0.886782\n",
      "[500]\ttraining's auc: 0.901892\tvalid_1's auc: 0.890645\n",
      "[600]\ttraining's auc: 0.905497\tvalid_1's auc: 0.894108\n",
      "[700]\ttraining's auc: 0.908766\tvalid_1's auc: 0.897228\n",
      "[800]\ttraining's auc: 0.911145\tvalid_1's auc: 0.8994\n",
      "[900]\ttraining's auc: 0.913386\tvalid_1's auc: 0.901457\n",
      "[1000]\ttraining's auc: 0.915675\tvalid_1's auc: 0.903478\n",
      "[1100]\ttraining's auc: 0.917675\tvalid_1's auc: 0.905022\n",
      "[1200]\ttraining's auc: 0.919488\tvalid_1's auc: 0.906541\n",
      "[1300]\ttraining's auc: 0.921365\tvalid_1's auc: 0.90798\n",
      "[1400]\ttraining's auc: 0.922944\tvalid_1's auc: 0.909263\n",
      "[1500]\ttraining's auc: 0.924403\tvalid_1's auc: 0.910498\n",
      "[1600]\ttraining's auc: 0.926072\tvalid_1's auc: 0.91185\n",
      "[1700]\ttraining's auc: 0.927582\tvalid_1's auc: 0.913005\n",
      "[1800]\ttraining's auc: 0.928867\tvalid_1's auc: 0.914006\n",
      "[1900]\ttraining's auc: 0.930107\tvalid_1's auc: 0.915007\n",
      "[2000]\ttraining's auc: 0.931292\tvalid_1's auc: 0.915848\n",
      "[2100]\ttraining's auc: 0.932432\tvalid_1's auc: 0.916698\n",
      "[2200]\ttraining's auc: 0.933627\tvalid_1's auc: 0.917547\n",
      "[2300]\ttraining's auc: 0.934611\tvalid_1's auc: 0.918424\n",
      "[2400]\ttraining's auc: 0.935614\tvalid_1's auc: 0.919266\n",
      "[2500]\ttraining's auc: 0.93664\tvalid_1's auc: 0.920067\n",
      "[2600]\ttraining's auc: 0.937584\tvalid_1's auc: 0.920868\n",
      "[2700]\ttraining's auc: 0.938473\tvalid_1's auc: 0.921628\n",
      "[2800]\ttraining's auc: 0.939437\tvalid_1's auc: 0.922377\n",
      "[2900]\ttraining's auc: 0.940295\tvalid_1's auc: 0.923102\n",
      "[3000]\ttraining's auc: 0.94115\tvalid_1's auc: 0.923763\n",
      "[3100]\ttraining's auc: 0.942044\tvalid_1's auc: 0.924587\n",
      "[3200]\ttraining's auc: 0.942807\tvalid_1's auc: 0.925286\n",
      "[3300]\ttraining's auc: 0.943546\tvalid_1's auc: 0.925938\n",
      "[3400]\ttraining's auc: 0.944164\tvalid_1's auc: 0.926509\n",
      "[3500]\ttraining's auc: 0.94488\tvalid_1's auc: 0.92712\n",
      "[3600]\ttraining's auc: 0.945444\tvalid_1's auc: 0.92762\n",
      "[3700]\ttraining's auc: 0.946118\tvalid_1's auc: 0.928146\n",
      "[3800]\ttraining's auc: 0.946769\tvalid_1's auc: 0.928694\n",
      "[3900]\ttraining's auc: 0.947423\tvalid_1's auc: 0.929252\n",
      "[4000]\ttraining's auc: 0.948021\tvalid_1's auc: 0.929808\n",
      "[4100]\ttraining's auc: 0.948594\tvalid_1's auc: 0.930247\n",
      "[4200]\ttraining's auc: 0.949172\tvalid_1's auc: 0.930706\n",
      "[4300]\ttraining's auc: 0.949775\tvalid_1's auc: 0.931183\n",
      "[4400]\ttraining's auc: 0.950277\tvalid_1's auc: 0.931586\n",
      "[4500]\ttraining's auc: 0.95092\tvalid_1's auc: 0.932161\n",
      "[4600]\ttraining's auc: 0.951429\tvalid_1's auc: 0.932609\n",
      "[4700]\ttraining's auc: 0.951956\tvalid_1's auc: 0.93305\n",
      "[4800]\ttraining's auc: 0.952485\tvalid_1's auc: 0.93355\n",
      "[4900]\ttraining's auc: 0.953074\tvalid_1's auc: 0.934043\n",
      "[5000]\ttraining's auc: 0.953655\tvalid_1's auc: 0.934544\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[5000]\ttraining's auc: 0.953655\tvalid_1's auc: 0.934544\n",
      "AUC: 0.9345440076527879\n",
      "Fold 3 finished in 0:05:19.855068\n",
      "Training on fold 4\n",
      "Training until validation scores don't improve for 150 rounds\n",
      "[100]\ttraining's auc: 0.868747\tvalid_1's auc: 0.873044\n",
      "[200]\ttraining's auc: 0.88249\tvalid_1's auc: 0.883936\n",
      "[300]\ttraining's auc: 0.890438\tvalid_1's auc: 0.890854\n",
      "[400]\ttraining's auc: 0.895867\tvalid_1's auc: 0.895345\n",
      "[500]\ttraining's auc: 0.900251\tvalid_1's auc: 0.898957\n",
      "[600]\ttraining's auc: 0.904047\tvalid_1's auc: 0.902443\n",
      "[700]\ttraining's auc: 0.906807\tvalid_1's auc: 0.904944\n",
      "[800]\ttraining's auc: 0.909494\tvalid_1's auc: 0.907269\n",
      "[900]\ttraining's auc: 0.911669\tvalid_1's auc: 0.909065\n",
      "[1000]\ttraining's auc: 0.914083\tvalid_1's auc: 0.910813\n",
      "[1100]\ttraining's auc: 0.916263\tvalid_1's auc: 0.912516\n",
      "[1200]\ttraining's auc: 0.91851\tvalid_1's auc: 0.914224\n",
      "[1300]\ttraining's auc: 0.920171\tvalid_1's auc: 0.915425\n",
      "[1400]\ttraining's auc: 0.922057\tvalid_1's auc: 0.916917\n",
      "[1500]\ttraining's auc: 0.923545\tvalid_1's auc: 0.918051\n",
      "[1600]\ttraining's auc: 0.92498\tvalid_1's auc: 0.919245\n",
      "[1700]\ttraining's auc: 0.926437\tvalid_1's auc: 0.920383\n",
      "[1800]\ttraining's auc: 0.927733\tvalid_1's auc: 0.921411\n",
      "[1900]\ttraining's auc: 0.928737\tvalid_1's auc: 0.922118\n",
      "[2000]\ttraining's auc: 0.929805\tvalid_1's auc: 0.922889\n",
      "[2100]\ttraining's auc: 0.930951\tvalid_1's auc: 0.92369\n",
      "[2200]\ttraining's auc: 0.931894\tvalid_1's auc: 0.924463\n",
      "[2300]\ttraining's auc: 0.932847\tvalid_1's auc: 0.925061\n",
      "[2400]\ttraining's auc: 0.933953\tvalid_1's auc: 0.925923\n",
      "[2500]\ttraining's auc: 0.934855\tvalid_1's auc: 0.926536\n",
      "[2600]\ttraining's auc: 0.935534\tvalid_1's auc: 0.926995\n",
      "[2700]\ttraining's auc: 0.936418\tvalid_1's auc: 0.927658\n",
      "[2800]\ttraining's auc: 0.937177\tvalid_1's auc: 0.928201\n",
      "[2900]\ttraining's auc: 0.937989\tvalid_1's auc: 0.928709\n",
      "[3000]\ttraining's auc: 0.938801\tvalid_1's auc: 0.929223\n",
      "[3100]\ttraining's auc: 0.939709\tvalid_1's auc: 0.929941\n",
      "[3200]\ttraining's auc: 0.940514\tvalid_1's auc: 0.930617\n",
      "[3300]\ttraining's auc: 0.941217\tvalid_1's auc: 0.931168\n",
      "[3400]\ttraining's auc: 0.94201\tvalid_1's auc: 0.931654\n",
      "[3500]\ttraining's auc: 0.942622\tvalid_1's auc: 0.932079\n",
      "[3600]\ttraining's auc: 0.943353\tvalid_1's auc: 0.932602\n",
      "[3700]\ttraining's auc: 0.944126\tvalid_1's auc: 0.933119\n",
      "[3800]\ttraining's auc: 0.944786\tvalid_1's auc: 0.933648\n",
      "[3900]\ttraining's auc: 0.945424\tvalid_1's auc: 0.934123\n",
      "[4000]\ttraining's auc: 0.946194\tvalid_1's auc: 0.934723\n",
      "[4100]\ttraining's auc: 0.946828\tvalid_1's auc: 0.935141\n",
      "[4200]\ttraining's auc: 0.947636\tvalid_1's auc: 0.935749\n",
      "[4300]\ttraining's auc: 0.948294\tvalid_1's auc: 0.936176\n",
      "[4400]\ttraining's auc: 0.94888\tvalid_1's auc: 0.936606\n",
      "[4500]\ttraining's auc: 0.949492\tvalid_1's auc: 0.93702\n",
      "[4600]\ttraining's auc: 0.950064\tvalid_1's auc: 0.937443\n",
      "[4700]\ttraining's auc: 0.950639\tvalid_1's auc: 0.937875\n",
      "[4800]\ttraining's auc: 0.95127\tvalid_1's auc: 0.938398\n",
      "[4900]\ttraining's auc: 0.951787\tvalid_1's auc: 0.938769\n",
      "[5000]\ttraining's auc: 0.952385\tvalid_1's auc: 0.939179\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[5000]\ttraining's auc: 0.952385\tvalid_1's auc: 0.939179\n",
      "AUC: 0.9391791480260098\n",
      "Fold 4 finished in 0:05:25.268286\n",
      "Training on fold 5\n",
      "Training until validation scores don't improve for 150 rounds\n",
      "[100]\ttraining's auc: 0.868172\tvalid_1's auc: 0.869408\n",
      "[200]\ttraining's auc: 0.881382\tvalid_1's auc: 0.881603\n",
      "[300]\ttraining's auc: 0.889418\tvalid_1's auc: 0.888715\n",
      "[400]\ttraining's auc: 0.895342\tvalid_1's auc: 0.89416\n",
      "[500]\ttraining's auc: 0.899829\tvalid_1's auc: 0.898221\n",
      "[600]\ttraining's auc: 0.903319\tvalid_1's auc: 0.9013\n",
      "[700]\ttraining's auc: 0.905866\tvalid_1's auc: 0.903495\n",
      "[800]\ttraining's auc: 0.908426\tvalid_1's auc: 0.905722\n",
      "[900]\ttraining's auc: 0.910678\tvalid_1's auc: 0.907587\n",
      "[1000]\ttraining's auc: 0.912609\tvalid_1's auc: 0.909135\n",
      "[1100]\ttraining's auc: 0.914701\tvalid_1's auc: 0.91096\n",
      "[1200]\ttraining's auc: 0.916762\tvalid_1's auc: 0.912694\n",
      "[1300]\ttraining's auc: 0.918667\tvalid_1's auc: 0.914406\n",
      "[1400]\ttraining's auc: 0.92058\tvalid_1's auc: 0.916252\n",
      "[1500]\ttraining's auc: 0.92238\tvalid_1's auc: 0.91765\n",
      "[1600]\ttraining's auc: 0.923971\tvalid_1's auc: 0.918997\n",
      "[1700]\ttraining's auc: 0.925413\tvalid_1's auc: 0.920046\n",
      "[1800]\ttraining's auc: 0.926625\tvalid_1's auc: 0.920961\n",
      "[1900]\ttraining's auc: 0.927957\tvalid_1's auc: 0.922074\n",
      "[2000]\ttraining's auc: 0.929084\tvalid_1's auc: 0.922807\n",
      "[2100]\ttraining's auc: 0.930328\tvalid_1's auc: 0.923853\n",
      "[2200]\ttraining's auc: 0.93161\tvalid_1's auc: 0.924961\n",
      "[2300]\ttraining's auc: 0.93272\tvalid_1's auc: 0.925759\n",
      "[2400]\ttraining's auc: 0.933837\tvalid_1's auc: 0.926533\n",
      "[2500]\ttraining's auc: 0.93464\tvalid_1's auc: 0.927083\n",
      "[2600]\ttraining's auc: 0.935818\tvalid_1's auc: 0.927954\n",
      "[2700]\ttraining's auc: 0.93663\tvalid_1's auc: 0.928525\n",
      "[2800]\ttraining's auc: 0.937753\tvalid_1's auc: 0.929296\n",
      "[2900]\ttraining's auc: 0.938585\tvalid_1's auc: 0.929866\n",
      "[3000]\ttraining's auc: 0.939458\tvalid_1's auc: 0.930456\n",
      "[3100]\ttraining's auc: 0.940276\tvalid_1's auc: 0.930962\n",
      "[3200]\ttraining's auc: 0.941231\tvalid_1's auc: 0.931632\n",
      "[3300]\ttraining's auc: 0.941971\tvalid_1's auc: 0.932087\n",
      "[3400]\ttraining's auc: 0.942932\tvalid_1's auc: 0.932715\n",
      "[3500]\ttraining's auc: 0.943717\tvalid_1's auc: 0.933283\n",
      "[3600]\ttraining's auc: 0.944456\tvalid_1's auc: 0.9338\n",
      "[3700]\ttraining's auc: 0.945112\tvalid_1's auc: 0.934208\n",
      "[3800]\ttraining's auc: 0.945905\tvalid_1's auc: 0.934781\n",
      "[3900]\ttraining's auc: 0.946686\tvalid_1's auc: 0.935335\n",
      "[4000]\ttraining's auc: 0.947383\tvalid_1's auc: 0.935819\n",
      "[4100]\ttraining's auc: 0.948007\tvalid_1's auc: 0.936222\n",
      "[4200]\ttraining's auc: 0.948657\tvalid_1's auc: 0.936672\n",
      "[4300]\ttraining's auc: 0.949332\tvalid_1's auc: 0.937118\n",
      "[4400]\ttraining's auc: 0.949997\tvalid_1's auc: 0.937589\n",
      "[4500]\ttraining's auc: 0.950592\tvalid_1's auc: 0.937955\n",
      "[4600]\ttraining's auc: 0.951165\tvalid_1's auc: 0.938376\n",
      "[4700]\ttraining's auc: 0.951693\tvalid_1's auc: 0.938683\n",
      "[4800]\ttraining's auc: 0.952233\tvalid_1's auc: 0.93905\n",
      "[4900]\ttraining's auc: 0.952766\tvalid_1's auc: 0.939386\n",
      "[5000]\ttraining's auc: 0.953297\tvalid_1's auc: 0.939725\n",
      "Did not meet early stopping. Best iteration is:\n",
      "[5000]\ttraining's auc: 0.953297\tvalid_1's auc: 0.939725\n",
      "AUC: 0.9397252530648554\n",
      "Fold 5 finished in 0:05:43.373843\n"
     ]
    }
   ],
   "source": [
    "model_lgb = CrossLgbBiClassifier()\n",
    "model_lgb.fit(train[used_features], train[target], tuning=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d99ae62b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "77cea055",
   "metadata": {},
   "source": [
    "## 查看模型特征重要性"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "2427c393",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:27:44.238821Z",
     "start_time": "2021-09-02T08:27:44.233272Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fimp = model_lgb.feature_importances_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "943cfbda",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:27:44.361484Z",
     "start_time": "2021-09-02T08:27:44.334135Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>feature</th>\n",
       "      <th>fold_1</th>\n",
       "      <th>fold_2</th>\n",
       "      <th>fold_3</th>\n",
       "      <th>fold_4</th>\n",
       "      <th>fold_5</th>\n",
       "      <th>average</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>card2</td>\n",
       "      <td>6121</td>\n",
       "      <td>5839</td>\n",
       "      <td>5580</td>\n",
       "      <td>5690</td>\n",
       "      <td>5822</td>\n",
       "      <td>5810.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>card1</td>\n",
       "      <td>4818</td>\n",
       "      <td>5083</td>\n",
       "      <td>4700</td>\n",
       "      <td>4615</td>\n",
       "      <td>4663</td>\n",
       "      <td>4775.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>addr1</td>\n",
       "      <td>4524</td>\n",
       "      <td>4697</td>\n",
       "      <td>4824</td>\n",
       "      <td>4639</td>\n",
       "      <td>4522</td>\n",
       "      <td>4641.2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>TransactionAmt</td>\n",
       "      <td>4139</td>\n",
       "      <td>4327</td>\n",
       "      <td>4293</td>\n",
       "      <td>4135</td>\n",
       "      <td>4140</td>\n",
       "      <td>4206.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C13</td>\n",
       "      <td>2851</td>\n",
       "      <td>2868</td>\n",
       "      <td>2943</td>\n",
       "      <td>2890</td>\n",
       "      <td>2772</td>\n",
       "      <td>2864.8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>395</th>\n",
       "      <td>V241</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>396</th>\n",
       "      <td>V305</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>397</th>\n",
       "      <td>V27</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>398</th>\n",
       "      <td>V89</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>399</th>\n",
       "      <td>id-32</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>400 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "            feature  fold_1  fold_2  fold_3  fold_4  fold_5  average\n",
       "0             card2    6121    5839    5580    5690    5822   5810.4\n",
       "1             card1    4818    5083    4700    4615    4663   4775.8\n",
       "2             addr1    4524    4697    4824    4639    4522   4641.2\n",
       "3    TransactionAmt    4139    4327    4293    4135    4140   4206.8\n",
       "4               C13    2851    2868    2943    2890    2772   2864.8\n",
       "..              ...     ...     ...     ...     ...     ...      ...\n",
       "395            V241       0       0       0       0       0      0.0\n",
       "396            V305       0       0       0       0       0      0.0\n",
       "397             V27       0       0       0       0       0      0.0\n",
       "398             V89       0       0       0       0       0      0.0\n",
       "399           id-32       0       0       0       0       0      0.0\n",
       "\n",
       "[400 rows x 7 columns]"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fimp"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "90832282",
   "metadata": {},
   "source": [
    "## 模型预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "7e5f58d5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:28:45.519777Z",
     "start_time": "2021-09-02T08:27:51.781937Z"
    }
   },
   "outputs": [],
   "source": [
    "predict_lgb = model_lgb.predict(test[used_features])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "0d13e4b5",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:14.161712Z",
     "start_time": "2021-09-02T08:29:14.156052Z"
    }
   },
   "outputs": [],
   "source": [
    "predict = predict_lgb"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "06bd0a9d",
   "metadata": {},
   "source": [
    "## 预测结果后处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "9c34f984",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:20.066682Z",
     "start_time": "2021-09-02T08:29:20.062201Z"
    }
   },
   "outputs": [],
   "source": [
    "from autox.process_data import clip_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "7d3d398b",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:21.260628Z",
     "start_time": "2021-09-02T08:29:21.252372Z"
    }
   },
   "outputs": [],
   "source": [
    "min_ = autox.info_['min_target']\n",
    "max_ = autox.info_['max_target']\n",
    "predict = clip_label(predict, min_, max_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "6abb804d",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:21.824166Z",
     "start_time": "2021-09-02T08:29:21.817408Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0, 1)"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "min_, max_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc891ec9",
   "metadata": {},
   "source": [
    "## 获取sub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "c3f7bc86",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:24.071306Z",
     "start_time": "2021-09-02T08:29:24.057223Z"
    }
   },
   "outputs": [],
   "source": [
    "sub = test[id_]\n",
    "sub[target] = predict\n",
    "sub.index = range(len(sub))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "7b15babd",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:25.812124Z",
     "start_time": "2021-09-02T08:29:25.797162Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>TransactionID</th>\n",
       "      <th>isFraud</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3663549</td>\n",
       "      <td>0.004643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3663550</td>\n",
       "      <td>0.003027</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3663551</td>\n",
       "      <td>0.004011</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3663552</td>\n",
       "      <td>0.001110</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3663553</td>\n",
       "      <td>0.003344</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   TransactionID   isFraud\n",
       "0        3663549  0.004643\n",
       "1        3663550  0.003027\n",
       "2        3663551  0.004011\n",
       "3        3663552  0.001110\n",
       "4        3663553  0.003344"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "id": "f5804dbc",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:26.486047Z",
     "start_time": "2021-09-02T08:29:26.480184Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(506691, 2)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "id": "4214a3e7",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:28.146058Z",
     "start_time": "2021-09-02T08:29:28.136958Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.9999280926655512, 5.87678498901363e-06)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub[target].max(), sub[target].min()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9873859c",
   "metadata": {},
   "source": [
    "## 输出结果"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "2dca42c9",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:39.377087Z",
     "start_time": "2021-09-02T08:29:39.373015Z"
    }
   },
   "outputs": [],
   "source": [
    "tag = \"ieee_lgb\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "e0016c8c",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2021-09-02T08:29:43.945563Z",
     "start_time": "2021-09-02T08:29:42.543292Z"
    },
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "path = f'./sub/sub_{tag}.csv'\n",
    "sub.to_csv(path, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee5a8b41",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "384px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
